git/watch-tool
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: use parser package for systemd logs

Browse source
This commit is contained in:
Patryk Hegenberg 2025-09-25 00:01:34 +02:00
parent 9aa1b7384d
commit e468b3a0e3
13 changed files with 897 additions and 275 deletions
Download patch file Download diff file Expand all files Collapse all files

3
file_monitor.go
View file

@ -7,6 +7,7 @@ import (
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"tixel_watch/helpers"
"tixel_watch/models" "tixel_watch/models"
"github.com/hpcloud/tail" "github.com/hpcloud/tail"
@ -175,7 +176,7 @@ func (p *NginxTJMLogParser) parseNginxTJM(entry models.LogMessage) models.LogMes
if len(parts) > 0 { if len(parts) > 0 {
timestampStr := strings.Trim(parts[0], "[]") timestampStr := strings.Trim(parts[0], "[]")
timestamp, err := parseRFC3339WithOptionalZ(timestampStr) timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
if err != nil { if err != nil {
slog.Error("unable to parse time", "error", err) slog.Error("unable to parse time", "error", err)
} }

86
helpers/helpers.go Normal file
View file

@ -0,0 +1,86 @@
package helpers
import (
"fmt"
"log/slog"
"os"
"regexp"
"strings"
"time"
"tixel_watch/models"
)
var (
syslogPattern = regexp.MustCompile(`^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) ([^\s]+) ([^:]+):\s*(.*)$`)
)
func ExtractSyslogHeader(line string) (models.SyslogFields, string) {
var syslogFields models.SyslogFields
matches := syslogPattern.FindStringSubmatch(strings.TrimSpace(line))
if len(matches) != 5 {
return syslogFields, line
}
sysTime, err := ParseSyslogTimeToRFC3339(matches[1])
if err != nil {
slog.Error("cant parse sys log time", "error", err)
}
syslogFields.SysLogTimestamp = sysTime
syslogFields.Hostname = matches[2]
syslogFields.ProcessInfo = matches[3]
return syslogFields, matches[4]
}
func GetNamedGroup(match []string, regex *regexp.Regexp, groupName string) string {
names := regex.SubexpNames()
for i, name := range names {
if name == groupName && i < len(match) {
return match[i]
}
}
return ""
}
func ParseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
timeStr += "Z"
}
return time.Parse(time.RFC3339Nano, timeStr)
}
var deToEnMonth = map[string]string{
"Jan": "Jan", "Feb": "Feb", "Mär": "Mar", "Apr": "Apr", "Mai": "May",
"Jun": "Jun", "Jul": "Jul", "Aug": "Aug", "Sep": "Sep", "Okt": "Oct",
"Nov": "Nov", "Dez": "Dec",
}
func translateMonth(syslogTime string) string {
for de, en := range deToEnMonth {
if strings.HasPrefix(syslogTime, de) {
return strings.Replace(syslogTime, de, en, 1)
}
}
return syslogTime
}
func ParseSyslogTimeToRFC3339(syslogTime string) (time.Time, error) {
const syslogLayout = "Jan 02 15:04:05"
syslogTime = translateMonth(syslogTime)
t, err := time.Parse(syslogLayout, syslogTime)
if err != nil {
return t, fmt.Errorf("cannot parse syslog time %q: %w", syslogTime, err)
}
now := time.Now()
t = t.AddDate(now.Year(), 0, 0)
return t, nil
}
func GetHostname() (string, error) {
hostname, err := os.Hostname()
if err != nil {
hostname = "unknown"
}
return hostname, nil
}

49
parser/am_parser.go Normal file
View file

@ -0,0 +1,49 @@
package parser
import (
"log/slog"
"regexp"
"strings"
"time"
"tixel_watch/helpers"
"tixel_watch/models"
)
var (
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
)
type AMParser struct{}
func (a *AMParser) Parse(line string) (models.LogMessage, error) {
newEntry := models.LogMessage{
Service: "access-manager",
}
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
newEntry.Host = syslogFields.Hostname
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
if len(matches) != 7 {
newEntry.Timestamp = time.Now()
newEntry.LogMessage = line
return newEntry, nil
}
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
if err != nil {
slog.Error("unable to parse time", "error", err)
return newEntry, err
}
baseInfo := models.AMBaseInfo{
ProcessID: matches[3],
ThreadID: strings.TrimSpace(matches[4]),
LoggerName: matches[5],
}
newEntry.Timestamp = timestamp
newEntry.LogLevel = matches[2]
newEntry.LogMessage = matches[6]
newEntry.ServiceInformation = baseInfo
return newEntry, nil
}

28
parser/default_parser.go Normal file
View file

@ -0,0 +1,28 @@
package parser
import (
"strings"
"time"
"tixel_watch/models"
)
type DefaultParser struct {
Service string
Tool string
}
func (d *DefaultParser) Parse(line string) (models.LogMessage, error) {
msg := models.LogMessage{
LogLevel: "unknown",
LogMessage: strings.TrimSpace(line),
Raw: line,
Timestamp: time.Now(),
}
if d.Service != "" {
msg.Service = d.Service
}
if d.Tool != "" {
msg.Tool = d.Tool
}
return msg, nil
}

27
parser/factory.go Normal file
View file

@ -0,0 +1,27 @@
package parser
func New(serviceName, logType string) (Parser, error) {
switch logType {
case "custom":
switch serviceName {
case "tixstream":
return &TSParser{}, nil
case "transfer-job-manager":
return &TJMParser{}, nil
case "access-manager":
return &AMParser{}, nil
case "tixel-control-center":
return &TCCParser{}, nil
case "nginx":
return &NginxParser{}, nil
case "nginx-tjm":
return &NginxTJMLogParser{ToolName: serviceName}, nil
default:
return &DefaultParser{Service: serviceName}, nil
}
case "json":
return &JSONParser{}, nil
default:
return &DefaultParser{Service: serviceName}, nil
}
}

19
parser/json_parser.go Normal file
View file

@ -0,0 +1,19 @@
package parser
import (
"encoding/json"
"log/slog"
"tixel_watch/models"
)
type JSONParser struct{}
func (j *JSONParser) Parse(line string) (models.LogMessage, error) {
var logMsg models.LogMessage
if err := json.Unmarshal([]byte(line), &logMsg); err == nil {
slog.Error("error parsing json line", "error", err)
return logMsg, err
}
return logMsg, nil
}

57
parser/nginx_parser.go Normal file
View file

@ -0,0 +1,57 @@
package parser
import (
"log/slog"
"regexp"
"strconv"
"strings"
"tixel_watch/models"
)
var (
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
)
type NginxParser struct{}
func (n *NginxParser) Parse(line string) (models.LogMessage, error) {
newEntry := models.LogMessage{
Service: "nginx",
}
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(line))
if len(matches) < 7 {
return newEntry, nil
}
statusCode, err := strconv.ParseInt(matches[5], 10, 64)
if err != nil {
slog.Error("cant parse statuscode", "error", err)
}
bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
if err != nil {
slog.Error("cant parse bytessend", "error", err)
}
baseInfo := models.NGinXBaseInfo{
ClientIP: matches[1],
RemoteUser: matches[2],
Request: matches[4],
StatusCode: int(statusCode),
BytesSend: int(bytesSend),
}
if len(matches) > 7 && matches[7] != "" {
baseInfo.Referer = matches[7]
}
if len(matches) > 8 && matches[8] != "" {
baseInfo.UserAgent = matches[8]
}
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
baseInfo.HTTPMethod = requestParts[0]
baseInfo.RequestURI = requestParts[1]
baseInfo.HTTPVersion = requestParts[2]
}
newEntry.ServiceInformation = baseInfo
return newEntry, nil
}

74
parser/nginx_tjm_parser.go Normal file
View file

@ -0,0 +1,74 @@
package parser
import (
"log/slog"
"strconv"
"strings"
"tixel_watch/helpers"
"tixel_watch/models"
)
type NginxTJMLogParser struct {
ToolName string
}
func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) {
entry := models.LogMessage{
Type: "log_entry",
Tool: p.ToolName,
Raw: line,
}
hostname, err := helpers.GetHostname()
if err != nil {
return entry, err
}
entry.Host = hostname
entry = p.parseNginxTJM(entry)
return entry, nil
}
func (p *NginxTJMLogParser) parseNginxTJM(entry models.LogMessage) models.LogMessage {
newEntry := entry
var nginxBase models.NGinXBaseInfo
parts := strings.Fields(entry.Raw)
if len(parts) < 10 {
return newEntry
}
if len(parts) > 0 {
timestampStr := strings.Trim(parts[0], "[]")
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
if err != nil {
slog.Error("unable to parse time", "error", err)
}
newEntry.Timestamp = timestamp
}
if len(parts) > 2 {
nginxBase.ClientIP = parts[2]
}
for i, part := range parts {
if strings.HasPrefix(part, "\"") {
if i+1 < len(parts) {
nginxBase.HTTPMethod = strings.Trim(part, "\"")
nginxBase.Route = parts[i+1]
}
break
}
}
for _, part := range parts {
if after, ok := strings.CutPrefix(part, "status="); ok {
statusCode, err := strconv.ParseInt(after, 10, 64)
if err != nil {
slog.Error("cant convert statuscode", "error", err)
}
nginxBase.StatusCode = int(statusCode)
break
}
}
newEntry.ServiceInformation = nginxBase
return newEntry
}

11
parser/parser.go Normal file
View file

@ -0,0 +1,11 @@
package parser
import (
"tixel_watch/models"
)
type Parser interface {
//TODO: Change parsers to return an error as well
Parse(line string) (models.LogMessage, error)
// Parse(line string) models.LogMessage
}

49
parser/tcc_parser.go Normal file
View file

@ -0,0 +1,49 @@
package parser
import (
"log/slog"
"regexp"
"strings"
"time"
"tixel_watch/helpers"
"tixel_watch/models"
)
var (
tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
)
type TCCParser struct{}
func (t *TCCParser) Parse(line string) (models.LogMessage, error) {
newEntry := models.LogMessage{
Service: "tixel-control-center",
}
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
newEntry.Host = syslogFields.Hostname
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
if len(matches) != 7 {
newEntry.Timestamp = time.Now()
newEntry.LogMessage = line
return newEntry, nil
}
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
if err != nil {
slog.Error("unable to parse time", "error", err)
return newEntry, err
}
baseInfo := models.TCCBaseInfo{
ProcessID: matches[3],
ThreadID: strings.TrimSpace(matches[4]),
LoggerName: matches[5],
}
newEntry.Timestamp = timestamp
newEntry.LogLevel = matches[2]
newEntry.LogMessage = matches[6]
newEntry.ServiceInformation = baseInfo
return newEntry, nil
}

91
parser/tjm_parser.go Normal file
View file

@ -0,0 +1,91 @@
package parser
import (
"log/slog"
"regexp"
"strings"
"tixel_watch/helpers"
"tixel_watch/models"
)
var (
tjmServicePattern = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (.*)$`)
tjmTransferIDPattern1 = regexp.MustCompile(`(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
tjmTransferIDPattern2 = regexp.MustCompile(`(?P<before>.*)(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
)
type TJMParser struct{}
func (t *TJMParser) Parse(line string) (models.LogMessage, error) {
newEntry := models.LogMessage{
Service: "transfer-job-manager",
}
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
newEntry.Host = syslogFields.Hostname
msg := strings.TrimSpace(logContent)
msg = strings.ReplaceAll(msg, " ", " ")
msg = strings.ReplaceAll(msg, "---", "")
msg = strings.ReplaceAll(msg, " ", " ")
parts := strings.Fields(msg)
if len(parts) < 4 {
newEntry.LogMessage = logContent
return newEntry, nil
}
matches := tjmServicePattern.FindStringSubmatch(logContent)
var baseInfo models.TJMTransferInfo
if len(matches) > 0 {
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
if err != nil {
slog.Error("unable to parse time", "error", err)
}
newEntry.Timestamp = timestamp
newEntry.LogLevel = strings.TrimSpace(matches[2])
newEntry.LogMessage = strings.TrimSpace(matches[8])
baseInfo = models.TJMTransferInfo{
ProcessID: strings.TrimSpace(matches[3]),
CorrelationID: strings.TrimSpace(matches[4]),
Username: strings.TrimSpace(matches[5]),
ThreadID: strings.TrimSpace(matches[6]),
JavaClass: strings.TrimSpace(matches[7]),
}
} else {
newEntry.LogMessage = logContent
}
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
var transferName string
var transferID string
if len(trNameMatch) > 0 {
transferName = trNameMatch[1]
newEntry.LogMessage = trNameMatch[2]
if strings.Contains(trNameMatch[1], "-in") {
baseInfo.Direction = "incoming"
}
if strings.Contains(trNameMatch[1], "-out") {
baseInfo.Direction = "outgoing"
}
}
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
if len(trIDMatch) > 0 {
transferID = trIDMatch[1]
}
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
if len(trIDMatch) > 0 {
transferID = trIDMatch[2]
}
if transferID != "" {
baseInfo.TransferID = transferID
} else if transferName != "" {
baseInfo.TransferID = transferName
} else {
baseInfo.TransferID = "no_transfer_id"
}
if baseInfo.StartTime.IsZero() {
baseInfo.StartTime = newEntry.Timestamp
}
newEntry.ServiceInformation = baseInfo
return newEntry, nil
}

136
parser/ts_parser.go Normal file
View file

@ -0,0 +1,136 @@
package parser
import (
"log/slog"
"regexp"
"strconv"
"strings"
"tixel_watch/helpers"
"tixel_watch/models"
)
var (
tsServicePattern = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
tsTransferIDPattern = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
tsDetailPattern1 = regexp.MustCompile(`in: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<dest>\S+) sender-id=(?P<sender_id>\S+)`)
tsDetailPattern2 = regexp.MustCompile(`out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s`)
tsDetailPattern3 = regexp.MustCompile(`out: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<src>\S+) receiver=(?P<receiver>\S+)`)
tsDetailPattern4 = regexp.MustCompile(`out: Start transfer (?P<thread>\d+/\d+), src=(?P<src>[^ ]*) dest=(?P<dest>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)`)
)
type TSParser struct{}
func (p *TSParser) Parse(line string) (models.LogMessage, error) {
newEntry := models.LogMessage{
Service: "tixstream",
}
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
newEntry.Host = syslogFields.Hostname
newEntry.Raw = line
newEntry.Type = "service_log"
matches := tsServicePattern.FindStringSubmatch(logContent)
if len(matches) > 0 {
timestampStr := strings.Join(strings.Split(matches[2], " "), "T")
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
if err != nil {
slog.Error("unable to parse time", "error", err)
}
if timestamp.IsZero() {
timestamp = syslogFields.SysLogTimestamp
}
newEntry.LogLevel = strings.TrimSpace(matches[1])
newEntry.LogLevel = strings.ReplaceAll(newEntry.LogLevel, "ACE_Message_Block", "")
newEntry.Timestamp = timestamp
newEntry.LogMessage = strings.TrimSpace(matches[3])
} else {
newEntry.LogMessage = logContent
}
var baseInfo models.TSTransferInfo
trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
var transferID string
if len(trNameMatch) > 0 {
transferID = trNameMatch[1]
newEntry.LogMessage = trNameMatch[2]
split := strings.Fields(trNameMatch[2])
switch split[0] {
case "in:":
baseInfo.Direction = "incoming"
case "out:":
baseInfo.Direction = "outgoing"
}
}
msg := strings.ReplaceAll(logContent, " ", " ")
parts := strings.Fields(msg)
if len(parts) < 5 {
return newEntry, nil
}
tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
baseInfo.Lane = threadInt
buffersInt, _ := strconv.Atoi(tsDetail[2])
fileCountInt, _ := strconv.Atoi(tsDetail[3])
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
streamsInt, _ := strconv.Atoi(tsDetail[6])
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
baseInfo.Buffers = buffersInt
baseInfo.FileCount = fileCountInt
baseInfo.FileSizeMB = fileSizeFloat
baseInfo.ChunkSize = chunkSizeInt
baseInfo.Streams = streamsInt
baseInfo.TargetDatarate = datarateFloat
baseInfo.Protocoll = tsDetail[8]
baseInfo.Dest = tsDetail[9]
baseInfo.SenderID = tsDetail[10]
}
tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 {
baseInfo.Target = tsDetail[1]
}
tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
baseInfo.Lane = threadInt
buffersInt, _ := strconv.Atoi(tsDetail[2])
baseInfo.Buffers = buffersInt
fileCountInt, _ := strconv.Atoi(tsDetail[3])
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
streamsInt, _ := strconv.Atoi(tsDetail[6])
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
baseInfo.FileCount = fileCountInt
baseInfo.FileSizeMB = fileSizeFloat
baseInfo.ChunkSize = chunkSizeInt
baseInfo.Streams = streamsInt
baseInfo.TargetDatarate = datarateFloat
baseInfo.Protocoll = tsDetail[8]
baseInfo.Src = tsDetail[9]
baseInfo.Receiver = tsDetail[10]
}
tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
baseInfo.Lane = threadInt
baseInfo.Src = tsDetail[2]
baseInfo.Dest = tsDetail[3]
}
if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
baseInfo.StartTime = newEntry.Timestamp
}
if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
baseInfo.EndTime = newEntry.Timestamp
}
if transferID != "" {
baseInfo.TransferID = transferID
} else {
baseInfo.TransferID = "no_transfer_id"
}
if !baseInfo.StartTime.IsZero() {
newEntry.ServiceInformation = baseInfo
}
return newEntry, nil
}

542
service_monitor.go
View file

@ -12,6 +12,7 @@ import (
"strings" "strings"
"time" "time"
"tixel_watch/models" "tixel_watch/models"
"tixel_watch/parser"
) )
type ServiceMonitor struct { type ServiceMonitor struct {
@ -207,20 +208,13 @@ func (jep *JournalEntryParser) extractSystemdFields(journalData map[string]any,
} }
func (jep *JournalEntryParser) parseServiceSpecific(entry models.LogMessage) models.LogMessage { func (jep *JournalEntryParser) parseServiceSpecific(entry models.LogMessage) models.LogMessage {
switch jep.serviceName { logParser, err := parser.New(jep.serviceName, "custom")
case "tixstream": if err != nil {
return parseTixstreamService(entry) slog.Error("cannot get service specific parser")
case "transfer-job-manager":
return parseTJMService(entry)
case "nginx":
return parseNginxService(entry)
case "access-manager":
return parseAMService(entry)
case "tixel-control-center":
return parseTCCService(entry)
default:
return entry return entry
} }
entry, err = logParser.Parse(entry.LogMessage)
return entry
} }
var ( var (
@ -239,283 +233,283 @@ var (
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`) nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
) )
func parseTixstreamService(entry models.LogMessage) models.LogMessage { // func parseTixstreamService(entry models.LogMessage) models.LogMessage {
newEntry := entry // newEntry := entry
var baseInfo models.TSTransferInfo // var baseInfo models.TSTransferInfo
matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage) // matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
if len(matches) > 0 { // if len(matches) > 0 {
timestamp := strings.Join(strings.Split(matches[2], " "), "T") // timestamp := strings.Join(strings.Split(matches[2], " "), "T")
newEntry.LogLevel = strings.TrimSpace(matches[1]) // newEntry.LogLevel = strings.TrimSpace(matches[1])
if newEntry.Timestamp.IsZero() { // if newEntry.Timestamp.IsZero() {
timeParsed, err := parseRFC3339WithOptionalZ(timestamp) // timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
if err != nil { // if err != nil {
slog.Error("cant parse time string", "error", err) // slog.Error("cant parse time string", "error", err)
} // }
newEntry.Timestamp = timeParsed // newEntry.Timestamp = timeParsed
} // }
newEntry.LogMessage = strings.TrimSpace(matches[3]) // newEntry.LogMessage = strings.TrimSpace(matches[3])
} // }
trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage) // trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
var transferID string // var transferID string
if len(trNameMatch) > 0 { // if len(trNameMatch) > 0 {
transferID = trNameMatch[1] // transferID = trNameMatch[1]
newEntry.LogMessage = trNameMatch[2] // newEntry.LogMessage = trNameMatch[2]
split := strings.Fields(trNameMatch[2]) // split := strings.Fields(trNameMatch[2])
switch split[0] { // switch split[0] {
case "in:": // case "in:":
baseInfo.Direction = "incoming" // baseInfo.Direction = "incoming"
case "out:": // case "out:":
baseInfo.Direction = "outgoing" // baseInfo.Direction = "outgoing"
} // }
} // }
msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ") // msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ")
parts := strings.Fields(msg) // parts := strings.Fields(msg)
if len(parts) < 5 { // if len(parts) < 5 {
return newEntry // return newEntry
} // }
tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage) // tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { // if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0]) // threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
buffersInt, _ := strconv.Atoi(tsDetail[2]) // buffersInt, _ := strconv.Atoi(tsDetail[2])
fileCountInt, _ := strconv.Atoi(tsDetail[3]) // fileCountInt, _ := strconv.Atoi(tsDetail[3])
chunkSizeInt, _ := strconv.Atoi(tsDetail[5]) // chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
streamsInt, _ := strconv.Atoi(tsDetail[6]) // streamsInt, _ := strconv.Atoi(tsDetail[6])
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64) // datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64) // fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
baseInfo.Lane = threadInt // baseInfo.Lane = threadInt
baseInfo.Buffers = buffersInt // baseInfo.Buffers = buffersInt
baseInfo.FileCount = fileCountInt // baseInfo.FileCount = fileCountInt
baseInfo.FileSizeMB = fileSizeFloat // baseInfo.FileSizeMB = fileSizeFloat
baseInfo.ChunkSize = chunkSizeInt // baseInfo.ChunkSize = chunkSizeInt
baseInfo.Streams = streamsInt // baseInfo.Streams = streamsInt
baseInfo.TargetDatarate = datarateFloat // baseInfo.TargetDatarate = datarateFloat
baseInfo.Protocoll = tsDetail[8] // baseInfo.Protocoll = tsDetail[8]
baseInfo.Dest = tsDetail[9] // baseInfo.Dest = tsDetail[9]
baseInfo.SenderID = tsDetail[10] // baseInfo.SenderID = tsDetail[10]
} // }
tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage) // tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { // if len(tsDetail) > 0 {
baseInfo.Target = tsDetail[1] // baseInfo.Target = tsDetail[1]
} // }
tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage) // tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { // if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0]) // threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
buffersInt, _ := strconv.Atoi(tsDetail[2]) // buffersInt, _ := strconv.Atoi(tsDetail[2])
fileCountInt, _ := strconv.Atoi(tsDetail[3]) // fileCountInt, _ := strconv.Atoi(tsDetail[3])
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64) // fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
chunkSizeInt, _ := strconv.Atoi(tsDetail[5]) // chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
streamsInt, _ := strconv.Atoi(tsDetail[6]) // streamsInt, _ := strconv.Atoi(tsDetail[6])
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64) // datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
baseInfo.Lane = threadInt // baseInfo.Lane = threadInt
baseInfo.Buffers = buffersInt // baseInfo.Buffers = buffersInt
baseInfo.FileCount = fileCountInt // baseInfo.FileCount = fileCountInt
baseInfo.FileSizeMB = fileSizeFloat // baseInfo.FileSizeMB = fileSizeFloat
baseInfo.ChunkSize = chunkSizeInt // baseInfo.ChunkSize = chunkSizeInt
baseInfo.Streams = streamsInt // baseInfo.Streams = streamsInt
baseInfo.TargetDatarate = datarateFloat // baseInfo.TargetDatarate = datarateFloat
baseInfo.Protocoll = tsDetail[8] // baseInfo.Protocoll = tsDetail[8]
baseInfo.Src = tsDetail[9] // baseInfo.Src = tsDetail[9]
baseInfo.Receiver = tsDetail[10] // baseInfo.Receiver = tsDetail[10]
} // }
tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage) // tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { // if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0]) // threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
baseInfo.Lane = threadInt // baseInfo.Lane = threadInt
baseInfo.Src = tsDetail[2] // baseInfo.Src = tsDetail[2]
baseInfo.Dest = tsDetail[3] // baseInfo.Dest = tsDetail[3]
} // }
if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") { // if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
baseInfo.StartTime = newEntry.Timestamp // baseInfo.StartTime = newEntry.Timestamp
} else { // } else {
baseInfo.StartTime = time.Now() // baseInfo.StartTime = time.Now()
} // }
if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") { // if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
baseInfo.EndTime = newEntry.Timestamp // baseInfo.EndTime = newEntry.Timestamp
} else { // } else {
baseInfo.EndTime = baseInfo.StartTime // baseInfo.EndTime = baseInfo.StartTime
} // }
if transferID != "" { // if transferID != "" {
baseInfo.TransferID = transferID // baseInfo.TransferID = transferID
} else { // } else {
baseInfo.TransferID = "no_transfer_id" // baseInfo.TransferID = "no_transfer_id"
} // }
newEntry.ServiceInformation = baseInfo // newEntry.ServiceInformation = baseInfo
return newEntry // return newEntry
} // }
func parseTJMService(entry models.LogMessage) models.LogMessage { // func parseTJMService(entry models.LogMessage) models.LogMessage {
newEntry := entry // newEntry := entry
var baseInfo models.TJMTransferInfo // var baseInfo models.TJMTransferInfo
logContent := entry.LogMessage // logContent := entry.LogMessage
msg := strings.TrimSpace(logContent) // msg := strings.TrimSpace(logContent)
msg = strings.ReplaceAll(msg, " ", " ") // msg = strings.ReplaceAll(msg, " ", " ")
msg = strings.ReplaceAll(msg, "---", "") // msg = strings.ReplaceAll(msg, "---", "")
msg = strings.ReplaceAll(msg, " ", " ") // msg = strings.ReplaceAll(msg, " ", " ")
parts := strings.Fields(msg) // parts := strings.Fields(msg)
if len(parts) < 4 { // if len(parts) < 4 {
return newEntry // return newEntry
} // }
matches := tjmServicePattern.FindStringSubmatch(logContent) // matches := tjmServicePattern.FindStringSubmatch(logContent)
if len(matches) > 0 { // if len(matches) > 0 {
timestamp := strings.Join(strings.Split(matches[2], " "), "T") // timestamp := strings.Join(strings.Split(matches[2], " "), "T")
newEntry.LogLevel = strings.TrimSpace(matches[1]) // newEntry.LogLevel = strings.TrimSpace(matches[1])
if newEntry.Timestamp.IsZero() { // if newEntry.Timestamp.IsZero() {
timeParsed, err := parseRFC3339WithOptionalZ(timestamp) // timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
if err != nil { // if err != nil {
slog.Error("cant parse time string", "error", err) // slog.Error("cant parse time string", "error", err)
} // }
newEntry.Timestamp = timeParsed // newEntry.Timestamp = timeParsed
} // }
newEntry.LogLevel = strings.TrimSpace(matches[2]) // newEntry.LogLevel = strings.TrimSpace(matches[2])
newEntry.LogMessage = strings.TrimSpace(matches[8]) // newEntry.LogMessage = strings.TrimSpace(matches[8])
baseInfo = models.TJMTransferInfo{ // baseInfo = models.TJMTransferInfo{
ProcessID: strings.TrimSpace(matches[3]), // ProcessID: strings.TrimSpace(matches[3]),
CorrelationID: strings.TrimSpace(matches[4]), // CorrelationID: strings.TrimSpace(matches[4]),
Username: strings.TrimSpace(matches[5]), // Username: strings.TrimSpace(matches[5]),
ThreadID: strings.TrimSpace(matches[6]), // ThreadID: strings.TrimSpace(matches[6]),
JavaClass: strings.TrimSpace(matches[7]), // JavaClass: strings.TrimSpace(matches[7]),
} // }
} else { // } else {
newEntry.LogMessage = logContent // newEntry.LogMessage = logContent
} // }
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage) // trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
var transferName string // var transferName string
var transferID string // var transferID string
if len(trNameMatch) > 0 { // if len(trNameMatch) > 0 {
transferName = trNameMatch[1] // transferName = trNameMatch[1]
newEntry.LogMessage = trNameMatch[2] // newEntry.LogMessage = trNameMatch[2]
if strings.Contains(trNameMatch[1], "-in") { // if strings.Contains(trNameMatch[1], "-in") {
baseInfo.Direction = "incoming" // baseInfo.Direction = "incoming"
} // }
if strings.Contains(trNameMatch[1], "-out") { // if strings.Contains(trNameMatch[1], "-out") {
baseInfo.Direction = "outgoing" // baseInfo.Direction = "outgoing"
} // }
} // }
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage) // trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
if len(trIDMatch) > 0 { // if len(trIDMatch) > 0 {
transferID = trIDMatch[1] // transferID = trIDMatch[1]
} // }
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage) // trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
if len(trIDMatch) > 0 { // if len(trIDMatch) > 0 {
transferID = trIDMatch[2] // transferID = trIDMatch[2]
} // }
if transferID != "" { // if transferID != "" {
baseInfo.TransferID = transferID // baseInfo.TransferID = transferID
} else if transferName != "" { // } else if transferName != "" {
baseInfo.TransferID = transferName // baseInfo.TransferID = transferName
} else { // } else {
baseInfo.TransferID = "no_transfer_id" // baseInfo.TransferID = "no_transfer_id"
} // }
baseInfo.StartTime = newEntry.Timestamp // baseInfo.StartTime = newEntry.Timestamp
baseInfo.StartTime = newEntry.Timestamp // baseInfo.StartTime = newEntry.Timestamp
newEntry.ServiceInformation = baseInfo // newEntry.ServiceInformation = baseInfo
return newEntry // return newEntry
} // }
func parseAMService(entry models.LogMessage) models.LogMessage { // func parseAMService(entry models.LogMessage) models.LogMessage {
newEntry := entry // newEntry := entry
logContent := newEntry.LogMessage // logContent := newEntry.LogMessage
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent)) // matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
if len(matches) != 7 { // if len(matches) != 7 {
return newEntry // return newEntry
} // }
timestampStr := strings.Join(strings.Split(matches[1], " "), "T") // timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
if newEntry.Timestamp.IsZero() { // if newEntry.Timestamp.IsZero() {
timeParsed, err := parseRFC3339WithOptionalZ(timestampStr) // timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
if err != nil { // if err != nil {
slog.Error("cant parse time string", "error", err) // slog.Error("cant parse time string", "error", err)
} // }
newEntry.Timestamp = timeParsed // newEntry.Timestamp = timeParsed
} // }
baseInfo := models.AMBaseInfo{ // baseInfo := models.AMBaseInfo{
ProcessID: matches[3], // ProcessID: matches[3],
ThreadID: strings.TrimSpace(matches[4]), // ThreadID: strings.TrimSpace(matches[4]),
LoggerName: matches[5], // LoggerName: matches[5],
} // }
newEntry.LogLevel = matches[2] // newEntry.LogLevel = matches[2]
newEntry.LogMessage = matches[6] // newEntry.LogMessage = matches[6]
newEntry.ServiceInformation = baseInfo // newEntry.ServiceInformation = baseInfo
return newEntry // return newEntry
} // }
func parseTCCService(entry models.LogMessage) models.LogMessage { // func parseTCCService(entry models.LogMessage) models.LogMessage {
newEntry := entry // newEntry := entry
logContent := newEntry.LogMessage // logContent := newEntry.LogMessage
matches := tccServicePattern.FindStringSubmatch(logContent) // matches := tccServicePattern.FindStringSubmatch(logContent)
if len(matches) != 7 { // if len(matches) != 7 {
return newEntry // return newEntry
} // }
timestampStr := strings.Join(strings.Split(matches[1], " "), "T") // timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
if newEntry.Timestamp.IsZero() { // if newEntry.Timestamp.IsZero() {
timeParsed, err := parseRFC3339WithOptionalZ(timestampStr) // timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
if err != nil { // if err != nil {
slog.Error("cant parse time string", "error", err) // slog.Error("cant parse time string", "error", err)
} // }
newEntry.Timestamp = timeParsed // newEntry.Timestamp = timeParsed
} // }
baseInfo := models.TCCBaseInfo{ // baseInfo := models.TCCBaseInfo{
ProcessID: matches[3], // ProcessID: matches[3],
ThreadID: strings.TrimSpace(matches[4]), // ThreadID: strings.TrimSpace(matches[4]),
LoggerName: matches[5], // LoggerName: matches[5],
} // }
newEntry.LogLevel = matches[2] // newEntry.LogLevel = matches[2]
newEntry.LogMessage = matches[6] // newEntry.LogMessage = matches[6]
newEntry.ServiceInformation = baseInfo // newEntry.ServiceInformation = baseInfo
return newEntry // return newEntry
} // }
func parseNginxService(entry models.LogMessage) models.LogMessage { // func parseNginxService(entry models.LogMessage) models.LogMessage {
newEntry := entry // newEntry := entry
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage)) // matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
if len(matches) < 7 { // if len(matches) < 7 {
return newEntry // return newEntry
} // }
statusCode, err := strconv.ParseInt(matches[5], 10, 64) // statusCode, err := strconv.ParseInt(matches[5], 10, 64)
if err != nil { // if err != nil {
slog.Error("cant parse statuscode", "error", err) // slog.Error("cant parse statuscode", "error", err)
} // }
bytesSend, err := strconv.ParseInt(matches[6], 10, 64) // bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
if err != nil { // if err != nil {
slog.Error("cant parse bytessend", "error", err) // slog.Error("cant parse bytessend", "error", err)
} // }
baseInfo := models.NGinXBaseInfo{ // baseInfo := models.NGinXBaseInfo{
ClientIP: matches[1], // ClientIP: matches[1],
RemoteUser: matches[2], // RemoteUser: matches[2],
Request: matches[4], // Request: matches[4],
StatusCode: int(statusCode), // StatusCode: int(statusCode),
BytesSend: int(bytesSend), // BytesSend: int(bytesSend),
} // }
if len(matches) > 7 && matches[7] != "" { // if len(matches) > 7 && matches[7] != "" {
baseInfo.Referer = matches[7] // baseInfo.Referer = matches[7]
} // }
if len(matches) > 8 && matches[8] != "" { // if len(matches) > 8 && matches[8] != "" {
baseInfo.UserAgent = matches[8] // baseInfo.UserAgent = matches[8]
} // }
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 { // if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
baseInfo.HTTPMethod = requestParts[0] // baseInfo.HTTPMethod = requestParts[0]
baseInfo.RequestURI = requestParts[1] // baseInfo.RequestURI = requestParts[1]
baseInfo.HTTPVersion = requestParts[2] // baseInfo.HTTPVersion = requestParts[2]
} // }
newEntry.ServiceInformation = baseInfo // newEntry.ServiceInformation = baseInfo
return newEntry // return newEntry
} // }
func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) { // func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") { // if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
timeStr += "Z" // timeStr += "Z"
} // }
return time.Parse(time.RFC3339Nano, timeStr) // return time.Parse(time.RFC3339Nano, timeStr)
} // }