From e468b3a0e3b6d8fb453838d44f5ff2fbea36b120 Mon Sep 17 00:00:00 2001 From: Patryk Hegenberg Date: Thu, 25 Sep 2025 00:01:34 +0200 Subject: [PATCH] refactor: use parser package for systemd logs --- file_monitor.go | 3 +- helpers/helpers.go | 86 ++++++ parser/am_parser.go | 49 ++++ parser/default_parser.go | 28 ++ parser/factory.go | 27 ++ parser/json_parser.go | 19 ++ parser/nginx_parser.go | 57 ++++ parser/nginx_tjm_parser.go | 74 +++++ parser/parser.go | 11 + parser/tcc_parser.go | 49 ++++ parser/tjm_parser.go | 91 +++++++ parser/ts_parser.go | 136 ++++++++++ service_monitor.go | 542 ++++++++++++++++++------------------- 13 files changed, 897 insertions(+), 275 deletions(-) create mode 100644 helpers/helpers.go create mode 100644 parser/am_parser.go create mode 100644 parser/default_parser.go create mode 100644 parser/factory.go create mode 100644 parser/json_parser.go create mode 100644 parser/nginx_parser.go create mode 100644 parser/nginx_tjm_parser.go create mode 100644 parser/parser.go create mode 100644 parser/tcc_parser.go create mode 100644 parser/tjm_parser.go create mode 100644 parser/ts_parser.go diff --git a/file_monitor.go b/file_monitor.go index cec14fb..09d40e6 100644 --- a/file_monitor.go +++ b/file_monitor.go @@ -7,6 +7,7 @@ import ( "regexp" "strconv" "strings" + "tixel_watch/helpers" "tixel_watch/models" "github.com/hpcloud/tail" @@ -175,7 +176,7 @@ func (p *NginxTJMLogParser) parseNginxTJM(entry models.LogMessage) models.LogMes if len(parts) > 0 { timestampStr := strings.Trim(parts[0], "[]") - timestamp, err := parseRFC3339WithOptionalZ(timestampStr) + timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr) if err != nil { slog.Error("unable to parse time", "error", err) } diff --git a/helpers/helpers.go b/helpers/helpers.go new file mode 100644 index 0000000..e16183c --- /dev/null +++ b/helpers/helpers.go @@ -0,0 +1,86 @@ +package helpers + +import ( + "fmt" + "log/slog" + "os" + "regexp" + "strings" + "time" + "tixel_watch/models" +) + +var ( + syslogPattern = regexp.MustCompile(`^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) ([^\s]+) ([^:]+):\s*(.*)$`) +) + +func ExtractSyslogHeader(line string) (models.SyslogFields, string) { + var syslogFields models.SyslogFields + + matches := syslogPattern.FindStringSubmatch(strings.TrimSpace(line)) + if len(matches) != 5 { + return syslogFields, line + } + + sysTime, err := ParseSyslogTimeToRFC3339(matches[1]) + if err != nil { + slog.Error("cant parse sys log time", "error", err) + } + syslogFields.SysLogTimestamp = sysTime + syslogFields.Hostname = matches[2] + syslogFields.ProcessInfo = matches[3] + + return syslogFields, matches[4] +} + +func GetNamedGroup(match []string, regex *regexp.Regexp, groupName string) string { + names := regex.SubexpNames() + for i, name := range names { + if name == groupName && i < len(match) { + return match[i] + } + } + return "" +} + +func ParseRFC3339WithOptionalZ(timeStr string) (time.Time, error) { + if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") { + timeStr += "Z" + } + return time.Parse(time.RFC3339Nano, timeStr) +} + +var deToEnMonth = map[string]string{ + "Jan": "Jan", "Feb": "Feb", "Mär": "Mar", "Apr": "Apr", "Mai": "May", + "Jun": "Jun", "Jul": "Jul", "Aug": "Aug", "Sep": "Sep", "Okt": "Oct", + "Nov": "Nov", "Dez": "Dec", +} + +func translateMonth(syslogTime string) string { + for de, en := range deToEnMonth { + if strings.HasPrefix(syslogTime, de) { + return strings.Replace(syslogTime, de, en, 1) + } + } + return syslogTime +} + +func ParseSyslogTimeToRFC3339(syslogTime string) (time.Time, error) { + const syslogLayout = "Jan 02 15:04:05" + syslogTime = translateMonth(syslogTime) + t, err := time.Parse(syslogLayout, syslogTime) + if err != nil { + return t, fmt.Errorf("cannot parse syslog time %q: %w", syslogTime, err) + } + now := time.Now() + t = t.AddDate(now.Year(), 0, 0) + return t, nil +} + +func GetHostname() (string, error) { + hostname, err := os.Hostname() + if err != nil { + hostname = "unknown" + } + return hostname, nil +} diff --git a/parser/am_parser.go b/parser/am_parser.go new file mode 100644 index 0000000..cfd9950 --- /dev/null +++ b/parser/am_parser.go @@ -0,0 +1,49 @@ +package parser + +import ( + "log/slog" + "regexp" + "strings" + "time" + "tixel_watch/helpers" + "tixel_watch/models" +) + +var ( + amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`) +) + +type AMParser struct{} + +func (a *AMParser) Parse(line string) (models.LogMessage, error) { + newEntry := models.LogMessage{ + Service: "access-manager", + } + syslogFields, logContent := helpers.ExtractSyslogHeader(line) + newEntry.Host = syslogFields.Hostname + + matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent)) + if len(matches) != 7 { + newEntry.Timestamp = time.Now() + newEntry.LogMessage = line + return newEntry, nil + } + + timestampStr := strings.Join(strings.Split(matches[1], " "), "T") + timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr) + if err != nil { + slog.Error("unable to parse time", "error", err) + return newEntry, err + } + baseInfo := models.AMBaseInfo{ + ProcessID: matches[3], + ThreadID: strings.TrimSpace(matches[4]), + LoggerName: matches[5], + } + newEntry.Timestamp = timestamp + newEntry.LogLevel = matches[2] + newEntry.LogMessage = matches[6] + newEntry.ServiceInformation = baseInfo + + return newEntry, nil +} diff --git a/parser/default_parser.go b/parser/default_parser.go new file mode 100644 index 0000000..664fc52 --- /dev/null +++ b/parser/default_parser.go @@ -0,0 +1,28 @@ +package parser + +import ( + "strings" + "time" + "tixel_watch/models" +) + +type DefaultParser struct { + Service string + Tool string +} + +func (d *DefaultParser) Parse(line string) (models.LogMessage, error) { + msg := models.LogMessage{ + LogLevel: "unknown", + LogMessage: strings.TrimSpace(line), + Raw: line, + Timestamp: time.Now(), + } + if d.Service != "" { + msg.Service = d.Service + } + if d.Tool != "" { + msg.Tool = d.Tool + } + return msg, nil +} diff --git a/parser/factory.go b/parser/factory.go new file mode 100644 index 0000000..fb6bc15 --- /dev/null +++ b/parser/factory.go @@ -0,0 +1,27 @@ +package parser + +func New(serviceName, logType string) (Parser, error) { + switch logType { + case "custom": + switch serviceName { + case "tixstream": + return &TSParser{}, nil + case "transfer-job-manager": + return &TJMParser{}, nil + case "access-manager": + return &arser{}, nil + case "tixel-control-center": + return &TCCParser{}, nil + case "nginx": + return &NginxParser{}, nil + case "nginx-tjm": + return &NginxTJMLogParser{ToolName: serviceName}, nil + default: + return &DefaultParser{Service: serviceName}, nil + } + case "json": + return &JSONParser{}, nil + default: + return &DefaultParser{Service: serviceName}, nil + } +} diff --git a/parser/json_parser.go b/parser/json_parser.go new file mode 100644 index 0000000..fd3527e --- /dev/null +++ b/parser/json_parser.go @@ -0,0 +1,19 @@ +package parser + +import ( + "encoding/json" + "log/slog" + "tixel_watch/models" +) + +type JSONParser struct{} + +func (j *JSONParser) Parse(line string) (models.LogMessage, error) { + var logMsg models.LogMessage + + if err := json.Unmarshal([]byte(line), &logMsg); err == nil { + slog.Error("error parsing json line", "error", err) + return logMsg, err + } + return logMsg, nil +} diff --git a/parser/nginx_parser.go b/parser/nginx_parser.go new file mode 100644 index 0000000..ee4da40 --- /dev/null +++ b/parser/nginx_parser.go @@ -0,0 +1,57 @@ +package parser + +import ( + "log/slog" + "regexp" + "strconv" + "strings" + "tixel_watch/models" +) + +var ( + nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`) +) + +type NginxParser struct{} + +func (n *NginxParser) Parse(line string) (models.LogMessage, error) { + newEntry := models.LogMessage{ + Service: "nginx", + } + + matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(line)) + if len(matches) < 7 { + return newEntry, nil + } + statusCode, err := strconv.ParseInt(matches[5], 10, 64) + if err != nil { + slog.Error("cant parse statuscode", "error", err) + } + bytesSend, err := strconv.ParseInt(matches[6], 10, 64) + if err != nil { + slog.Error("cant parse bytessend", "error", err) + } + baseInfo := models.NGinXBaseInfo{ + ClientIP: matches[1], + RemoteUser: matches[2], + Request: matches[4], + StatusCode: int(statusCode), + BytesSend: int(bytesSend), + } + + if len(matches) > 7 && matches[7] != "" { + baseInfo.Referer = matches[7] + } + if len(matches) > 8 && matches[8] != "" { + baseInfo.UserAgent = matches[8] + } + + if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 { + baseInfo.HTTPMethod = requestParts[0] + baseInfo.RequestURI = requestParts[1] + baseInfo.HTTPVersion = requestParts[2] + } + newEntry.ServiceInformation = baseInfo + + return newEntry, nil +} diff --git a/parser/nginx_tjm_parser.go b/parser/nginx_tjm_parser.go new file mode 100644 index 0000000..d5f4f5e --- /dev/null +++ b/parser/nginx_tjm_parser.go @@ -0,0 +1,74 @@ +package parser + +import ( + "log/slog" + "strconv" + "strings" + "tixel_watch/helpers" + "tixel_watch/models" +) + +type NginxTJMLogParser struct { + ToolName string +} + +func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) { + entry := models.LogMessage{ + Type: "log_entry", + Tool: p.ToolName, + Raw: line, + } + hostname, err := helpers.GetHostname() + if err != nil { + return entry, err + } + entry.Host = hostname + entry = p.parseNginxTJM(entry) + return entry, nil +} + +func (p *NginxTJMLogParser) parseNginxTJM(entry models.LogMessage) models.LogMessage { + newEntry := entry + var nginxBase models.NGinXBaseInfo + parts := strings.Fields(entry.Raw) + if len(parts) < 10 { + return newEntry + } + + if len(parts) > 0 { + timestampStr := strings.Trim(parts[0], "[]") + timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr) + if err != nil { + slog.Error("unable to parse time", "error", err) + } + newEntry.Timestamp = timestamp + } + + if len(parts) > 2 { + nginxBase.ClientIP = parts[2] + } + + for i, part := range parts { + if strings.HasPrefix(part, "\"") { + if i+1 < len(parts) { + nginxBase.HTTPMethod = strings.Trim(part, "\"") + nginxBase.Route = parts[i+1] + } + break + } + } + + for _, part := range parts { + if after, ok := strings.CutPrefix(part, "status="); ok { + statusCode, err := strconv.ParseInt(after, 10, 64) + if err != nil { + slog.Error("cant convert statuscode", "error", err) + } + nginxBase.StatusCode = int(statusCode) + break + } + } + newEntry.ServiceInformation = nginxBase + + return newEntry +} diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..480c741 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,11 @@ +package parser + +import ( + "tixel_watch/models" +) + +type Parser interface { + //TODO: Change parsers to return an error as well + Parse(line string) (models.LogMessage, error) + // Parse(line string) models.LogMessage +} diff --git a/parser/tcc_parser.go b/parser/tcc_parser.go new file mode 100644 index 0000000..2c117c9 --- /dev/null +++ b/parser/tcc_parser.go @@ -0,0 +1,49 @@ +package parser + +import ( + "log/slog" + "regexp" + "strings" + "time" + "tixel_watch/helpers" + "tixel_watch/models" +) + +var ( + tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`) +) + +type TCCParser struct{} + +func (t *TCCParser) Parse(line string) (models.LogMessage, error) { + newEntry := models.LogMessage{ + Service: "tixel-control-center", + } + syslogFields, logContent := helpers.ExtractSyslogHeader(line) + newEntry.Host = syslogFields.Hostname + + matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(logContent)) + if len(matches) != 7 { + newEntry.Timestamp = time.Now() + newEntry.LogMessage = line + return newEntry, nil + } + + timestampStr := strings.Join(strings.Split(matches[1], " "), "T") + timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr) + if err != nil { + slog.Error("unable to parse time", "error", err) + return newEntry, err + } + baseInfo := models.TCCBaseInfo{ + ProcessID: matches[3], + ThreadID: strings.TrimSpace(matches[4]), + LoggerName: matches[5], + } + newEntry.Timestamp = timestamp + newEntry.LogLevel = matches[2] + newEntry.LogMessage = matches[6] + newEntry.ServiceInformation = baseInfo + + return newEntry, nil +} diff --git a/parser/tjm_parser.go b/parser/tjm_parser.go new file mode 100644 index 0000000..850e21c --- /dev/null +++ b/parser/tjm_parser.go @@ -0,0 +1,91 @@ +package parser + +import ( + "log/slog" + "regexp" + "strings" + "tixel_watch/helpers" + "tixel_watch/models" +) + +var ( + tjmServicePattern = regexp.MustCompile(`^(?