feat: delete old specific parsers and use new generic parser in file_monitor
This commit is contained in:
parent
0830b403e0
commit
794180c6ab
9 changed files with 21 additions and 595 deletions
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"watch-tool/models"
|
"watch-tool/models"
|
||||||
"watch-tool/parser"
|
"watch-tool/parser"
|
||||||
|
"watch-tool/patterns"
|
||||||
|
|
||||||
"github.com/hpcloud/tail"
|
"github.com/hpcloud/tail"
|
||||||
)
|
)
|
||||||
|
|
@ -22,22 +23,28 @@ func NewFileMonitor(config ToolConfig, hostname string) *FileMonitor {
|
||||||
var logParser parser.Parser
|
var logParser parser.Parser
|
||||||
|
|
||||||
if config.Format.Pattern != "" {
|
if config.Format.Pattern != "" {
|
||||||
pattern, err := regexp.Compile(config.Format.Pattern)
|
compiledRegex, err := regexp.Compile(config.Format.Pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("invalid regex pattern", "tool", config.Name, "error", err)
|
slog.Error("Invalid regex pattern in tool config", "tool", config.Name, "error", err)
|
||||||
logParser = &parser.DefaultParser{}
|
logParser = parser.NewGenericParser(config.Name, hostname)
|
||||||
} else {
|
} else {
|
||||||
logParser = &parser.RegexLogParser{
|
gp := parser.NewGenericParser(config.Name, hostname)
|
||||||
Pattern: pattern,
|
|
||||||
Fields: config.Format.Fields,
|
customExtractor := patterns.CompiledExtractor{
|
||||||
Toolname: config.Name,
|
Name: "config_custom_pattern",
|
||||||
|
Pattern: compiledRegex,
|
||||||
|
Fields: config.Format.Fields,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
gp.Extractors = append(gp.Extractors, customExtractor)
|
||||||
|
logParser = gp
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
var err error
|
var err error
|
||||||
logParser, err = parser.New(config.Name, "custom", hostname)
|
logParser, err = parser.New(config.Name, "custom", hostname)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("cannot get tool specific parser", "error", err)
|
slog.Error("Cannot get tool specific parser from factory", "error", err)
|
||||||
|
logParser = parser.NewGenericParser(config.Name, hostname)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -74,7 +81,7 @@ func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage)
|
||||||
}
|
}
|
||||||
|
|
||||||
if line.Err != nil {
|
if line.Err != nil {
|
||||||
slog.Error("error reading log file", "tool", fm.config.Name, "error", line.Err)
|
slog.Error("Error reading log file", "tool", fm.config.Name, "error", line.Err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -84,7 +91,11 @@ func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage)
|
||||||
|
|
||||||
entry, err := fm.parser.Parse(line.Text)
|
entry, err := fm.parser.Parse(line.Text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("error parsing log line", "error", err)
|
slog.Error("Error parsing log line", "tool", fm.config.Name, "error", err)
|
||||||
|
} else {
|
||||||
|
if entry.Tool == "" {
|
||||||
|
entry.Tool = fm.config.Name
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
|
|
|
||||||
|
|
@ -1,49 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"watch-tool/helpers"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
|
||||||
)
|
|
||||||
|
|
||||||
type AMParser struct{}
|
|
||||||
|
|
||||||
func (a *AMParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
newEntry := models.LogMessage{
|
|
||||||
Service: "access-manager",
|
|
||||||
}
|
|
||||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
|
||||||
newEntry.Host = syslogFields.Hostname
|
|
||||||
|
|
||||||
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
|
||||||
if len(matches) != 7 {
|
|
||||||
newEntry.Timestamp = time.Now()
|
|
||||||
newEntry.LogMessage = line
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
|
||||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("unable to parse time", "error", err)
|
|
||||||
return newEntry, err
|
|
||||||
}
|
|
||||||
baseInfo := models.AMBaseInfo{
|
|
||||||
ProcessID: matches[3],
|
|
||||||
ThreadID: strings.TrimSpace(matches[4]),
|
|
||||||
LoggerName: matches[5],
|
|
||||||
}
|
|
||||||
newEntry.Timestamp = timestamp
|
|
||||||
newEntry.LogLevel = matches[2]
|
|
||||||
newEntry.LogMessage = matches[6]
|
|
||||||
newEntry.ServiceInformation = baseInfo
|
|
||||||
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DefaultParser struct {
|
|
||||||
Service string
|
|
||||||
Tool string
|
|
||||||
Hostname string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *DefaultParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
msg := models.LogMessage{
|
|
||||||
LogLevel: "unknown",
|
|
||||||
LogMessage: strings.TrimSpace(line),
|
|
||||||
Raw: line,
|
|
||||||
Timestamp: time.Now(),
|
|
||||||
}
|
|
||||||
if d.Service != "" {
|
|
||||||
msg.Service = d.Service
|
|
||||||
}
|
|
||||||
if d.Tool != "" {
|
|
||||||
msg.Tool = d.Tool
|
|
||||||
}
|
|
||||||
return msg, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,117 +1,3 @@
|
||||||
// package parser
|
|
||||||
|
|
||||||
// import (
|
|
||||||
// "fmt"
|
|
||||||
// "strconv"
|
|
||||||
// "strings"
|
|
||||||
// "time"
|
|
||||||
// "watch-tool/models"
|
|
||||||
// "watch-tool/patterns"
|
|
||||||
// )
|
|
||||||
|
|
||||||
// type GenericParser struct {
|
|
||||||
// ServiceName string
|
|
||||||
// Hostname string
|
|
||||||
// Extractors []patterns.CompiledExtractor
|
|
||||||
// CommonExt []patterns.CompiledExtractor
|
|
||||||
// }
|
|
||||||
|
|
||||||
// func NewGenericParser(serviceName, hostname string) *GenericParser {
|
|
||||||
// repo := patterns.GetInstance()
|
|
||||||
// return &GenericParser{
|
|
||||||
// ServiceName: serviceName,
|
|
||||||
// Hostname: hostname,
|
|
||||||
// Extractors: repo.GetExtractors(serviceName),
|
|
||||||
// CommonExt: repo.GetExtractors("common"),
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// func (p *GenericParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
// entry := models.LogMessage{
|
|
||||||
// Service: p.ServiceName,
|
|
||||||
// Host: p.Hostname,
|
|
||||||
// Timestamp: time.Now(),
|
|
||||||
// Raw: line,
|
|
||||||
// Fields: make(map[string]any),
|
|
||||||
// }
|
|
||||||
|
|
||||||
// // 1. Common Extractors laufen lassen (z.B. Syslog Header entfernen/parsen)
|
|
||||||
// // Wir nutzen eine temporäre Variable für den Rest-String, falls Header entfernt werden soll
|
|
||||||
// currentLine := line
|
|
||||||
|
|
||||||
// // Hinweis: Hier könnte man Syslog-Logik generisch einbauen.
|
|
||||||
// // Fürs Erste wenden wir Pattern einfach auf die Zeile an.
|
|
||||||
|
|
||||||
// // 2. Service Extractors anwenden
|
|
||||||
// // Wir probieren ALLE Extractors, um maximale Informationen zu gewinnen.
|
|
||||||
// // Das simuliert die Logik deiner alten Parser (erst Header, dann Details).
|
|
||||||
// allExtractors := append(p.CommonExt, p.Extractors...)
|
|
||||||
|
|
||||||
// for _, ext := range allExtractors {
|
|
||||||
// matches := ext.Pattern.FindStringSubmatch(currentLine)
|
|
||||||
// if matches == nil {
|
|
||||||
// continue
|
|
||||||
// }
|
|
||||||
|
|
||||||
// subexpNames := ext.Pattern.SubexpNames()
|
|
||||||
// for i, matchValue := range matches {
|
|
||||||
// if i == 0 || matchValue == "" {
|
|
||||||
// continue
|
|
||||||
// }
|
|
||||||
|
|
||||||
// groupName := subexpNames[i]
|
|
||||||
// if groupName == "" {
|
|
||||||
// continue
|
|
||||||
// }
|
|
||||||
|
|
||||||
// targetType := ext.Fields[groupName]
|
|
||||||
// parsedValue, err := convertType(matchValue, targetType)
|
|
||||||
// if err == nil {
|
|
||||||
// switch groupName {
|
|
||||||
// case "timestamp":
|
|
||||||
// if t, ok := parsedValue.(time.Time); ok {
|
|
||||||
// entry.Timestamp = t
|
|
||||||
// }
|
|
||||||
// case "log_level":
|
|
||||||
// entry.LogLevel = fmt.Sprintf("%v", parsedValue)
|
|
||||||
// case "message":
|
|
||||||
// entry.LogMessage = fmt.Sprintf("%v", parsedValue)
|
|
||||||
// default:
|
|
||||||
// entry.Fields[groupName] = parsedValue
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// if entry.LogMessage == "" {
|
|
||||||
// entry.LogMessage = strings.TrimSpace(line)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// return entry, nil
|
|
||||||
// }
|
|
||||||
|
|
||||||
// func convertType(value, typeDef string) (any, error) {
|
|
||||||
// if strings.HasPrefix(typeDef, "int") {
|
|
||||||
// return strconv.Atoi(value)
|
|
||||||
// }
|
|
||||||
// if strings.HasPrefix(typeDef, "float") {
|
|
||||||
// return strconv.ParseFloat(value, 64)
|
|
||||||
// }
|
|
||||||
// if after, ok := strings.CutPrefix(typeDef, "time:"); ok {
|
|
||||||
// layout := after
|
|
||||||
// // Workaround für Syslog (Jahr fehlt oft), hier vereinfacht:
|
|
||||||
// if layout == "Jan 02 15:04:05" {
|
|
||||||
// t, err := time.Parse(layout, value)
|
|
||||||
// if err == nil {
|
|
||||||
// return t.AddDate(time.Now().Year(), 0, 0), nil
|
|
||||||
// }
|
|
||||||
// return t, err
|
|
||||||
// }
|
|
||||||
// return time.Parse(layout, value)
|
|
||||||
// }
|
|
||||||
// // Default: String
|
|
||||||
// return value, nil
|
|
||||||
// }
|
|
||||||
package parser
|
package parser
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -287,8 +173,6 @@ func (p *GenericParser) mapField(entry *models.LogMessage, key string, value any
|
||||||
entry.PID = pid
|
entry.PID = pid
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Mapping auf ServiceInformation Felder (Optional, falls nötig)
|
|
||||||
// case "transfer_id": ...
|
|
||||||
|
|
||||||
default:
|
default:
|
||||||
entry.Fields[key] = value
|
entry.Fields[key] = value
|
||||||
|
|
|
||||||
|
|
@ -1,57 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
|
||||||
)
|
|
||||||
|
|
||||||
type NginxParser struct{}
|
|
||||||
|
|
||||||
func (n *NginxParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
newEntry := models.LogMessage{
|
|
||||||
Service: "nginx",
|
|
||||||
}
|
|
||||||
|
|
||||||
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(line))
|
|
||||||
if len(matches) < 7 {
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
statusCode, err := strconv.ParseInt(matches[5], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("cant parse statuscode", "error", err)
|
|
||||||
}
|
|
||||||
bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("cant parse bytessend", "error", err)
|
|
||||||
}
|
|
||||||
baseInfo := models.NGinXBaseInfo{
|
|
||||||
ClientIP: matches[1],
|
|
||||||
RemoteUser: matches[2],
|
|
||||||
Request: matches[4],
|
|
||||||
StatusCode: int(statusCode),
|
|
||||||
BytesSend: int(bytesSend),
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) > 7 && matches[7] != "" {
|
|
||||||
baseInfo.Referer = matches[7]
|
|
||||||
}
|
|
||||||
if len(matches) > 8 && matches[8] != "" {
|
|
||||||
baseInfo.UserAgent = matches[8]
|
|
||||||
}
|
|
||||||
|
|
||||||
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
|
|
||||||
baseInfo.HTTPMethod = requestParts[0]
|
|
||||||
baseInfo.RequestURI = requestParts[1]
|
|
||||||
baseInfo.HTTPVersion = requestParts[2]
|
|
||||||
}
|
|
||||||
newEntry.ServiceInformation = baseInfo
|
|
||||||
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,58 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
type RegexLogParser struct {
|
|
||||||
Pattern *regexp.Regexp
|
|
||||||
Fields map[string]string
|
|
||||||
Toolname string
|
|
||||||
Hostname string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *RegexLogParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
entry := models.LogMessage{Type: "log_entry"}
|
|
||||||
entry.Tool = p.Toolname
|
|
||||||
entry.Raw = line
|
|
||||||
entry.Host = p.Hostname
|
|
||||||
|
|
||||||
fields := p.parseWithPattern(line)
|
|
||||||
if fields != nil {
|
|
||||||
entry.Fields = fields
|
|
||||||
} else {
|
|
||||||
entry.LogMessage = strings.TrimSpace(line)
|
|
||||||
}
|
|
||||||
|
|
||||||
return entry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *RegexLogParser) parseWithPattern(text string) map[string]any {
|
|
||||||
matches := p.Pattern.FindStringSubmatch(text)
|
|
||||||
if matches == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
fields := make(map[string]any)
|
|
||||||
subexpNames := p.Pattern.SubexpNames()
|
|
||||||
|
|
||||||
for i, match := range matches {
|
|
||||||
if i == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if i < len(subexpNames) && subexpNames[i] != "" {
|
|
||||||
fieldName := subexpNames[i]
|
|
||||||
|
|
||||||
if mappedName, exists := p.Fields[fieldName]; exists {
|
|
||||||
fieldName = mappedName
|
|
||||||
}
|
|
||||||
|
|
||||||
fields[fieldName] = match
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fields
|
|
||||||
}
|
|
||||||
|
|
@ -1,49 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"watch-tool/helpers"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
|
||||||
)
|
|
||||||
|
|
||||||
type TCCParser struct{}
|
|
||||||
|
|
||||||
func (t *TCCParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
newEntry := models.LogMessage{
|
|
||||||
Service: "tixel-control-center",
|
|
||||||
}
|
|
||||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
|
||||||
newEntry.Host = syslogFields.Hostname
|
|
||||||
|
|
||||||
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
|
||||||
if len(matches) != 7 {
|
|
||||||
newEntry.Timestamp = time.Now()
|
|
||||||
newEntry.LogMessage = line
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
|
||||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("unable to parse time", "error", err)
|
|
||||||
return newEntry, err
|
|
||||||
}
|
|
||||||
baseInfo := models.TCCBaseInfo{
|
|
||||||
ProcessID: matches[3],
|
|
||||||
ThreadID: strings.TrimSpace(matches[4]),
|
|
||||||
LoggerName: matches[5],
|
|
||||||
}
|
|
||||||
newEntry.Timestamp = timestamp
|
|
||||||
newEntry.LogLevel = matches[2]
|
|
||||||
newEntry.LogMessage = matches[6]
|
|
||||||
newEntry.ServiceInformation = baseInfo
|
|
||||||
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,91 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"watch-tool/helpers"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
tjmServicePattern = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
|
|
||||||
tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (.*)$`)
|
|
||||||
tjmTransferIDPattern1 = regexp.MustCompile(`(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
|
|
||||||
tjmTransferIDPattern2 = regexp.MustCompile(`(?P<before>.*)(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
|
|
||||||
)
|
|
||||||
|
|
||||||
type TJMParser struct{}
|
|
||||||
|
|
||||||
func (t *TJMParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
newEntry := models.LogMessage{
|
|
||||||
Service: "transfer-job-manager",
|
|
||||||
}
|
|
||||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
|
||||||
newEntry.Host = syslogFields.Hostname
|
|
||||||
|
|
||||||
msg := strings.TrimSpace(logContent)
|
|
||||||
msg = strings.ReplaceAll(msg, " ", " ")
|
|
||||||
msg = strings.ReplaceAll(msg, "---", "")
|
|
||||||
msg = strings.ReplaceAll(msg, " ", " ")
|
|
||||||
parts := strings.Fields(msg)
|
|
||||||
if len(parts) < 4 {
|
|
||||||
newEntry.LogMessage = logContent
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
matches := tjmServicePattern.FindStringSubmatch(logContent)
|
|
||||||
var baseInfo models.TJMTransferInfo
|
|
||||||
if len(matches) > 0 {
|
|
||||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
|
||||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("unable to parse time", "error", err)
|
|
||||||
}
|
|
||||||
newEntry.Timestamp = timestamp
|
|
||||||
newEntry.LogLevel = strings.TrimSpace(matches[2])
|
|
||||||
newEntry.LogMessage = strings.TrimSpace(matches[8])
|
|
||||||
baseInfo = models.TJMTransferInfo{
|
|
||||||
ProcessID: strings.TrimSpace(matches[3]),
|
|
||||||
CorrelationID: strings.TrimSpace(matches[4]),
|
|
||||||
Username: strings.TrimSpace(matches[5]),
|
|
||||||
ThreadID: strings.TrimSpace(matches[6]),
|
|
||||||
JavaClass: strings.TrimSpace(matches[7]),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
newEntry.LogMessage = logContent
|
|
||||||
}
|
|
||||||
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
var transferName string
|
|
||||||
var transferID string
|
|
||||||
if len(trNameMatch) > 0 {
|
|
||||||
transferName = trNameMatch[1]
|
|
||||||
newEntry.LogMessage = trNameMatch[2]
|
|
||||||
if strings.Contains(trNameMatch[1], "-in") {
|
|
||||||
baseInfo.Direction = "incoming"
|
|
||||||
}
|
|
||||||
if strings.Contains(trNameMatch[1], "-out") {
|
|
||||||
baseInfo.Direction = "outgoing"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
if len(trIDMatch) > 0 {
|
|
||||||
transferID = trIDMatch[1]
|
|
||||||
}
|
|
||||||
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
if len(trIDMatch) > 0 {
|
|
||||||
transferID = trIDMatch[2]
|
|
||||||
}
|
|
||||||
if transferID != "" {
|
|
||||||
baseInfo.TransferID = transferID
|
|
||||||
} else if transferName != "" {
|
|
||||||
baseInfo.TransferID = transferName
|
|
||||||
} else {
|
|
||||||
baseInfo.TransferID = "no_transfer_id"
|
|
||||||
}
|
|
||||||
if baseInfo.StartTime.IsZero() {
|
|
||||||
baseInfo.StartTime = newEntry.Timestamp
|
|
||||||
}
|
|
||||||
newEntry.ServiceInformation = baseInfo
|
|
||||||
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,136 +0,0 @@
|
||||||
package parser
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"watch-tool/helpers"
|
|
||||||
"watch-tool/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
tsServicePattern = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
|
|
||||||
tsTransferIDPattern = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
|
|
||||||
tsDetailPattern1 = regexp.MustCompile(`in: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<dest>\S+) sender-id=(?P<sender_id>\S+)`)
|
|
||||||
tsDetailPattern2 = regexp.MustCompile(`out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s`)
|
|
||||||
tsDetailPattern3 = regexp.MustCompile(`out: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<src>\S+) receiver=(?P<receiver>\S+)`)
|
|
||||||
tsDetailPattern4 = regexp.MustCompile(`out: Start transfer (?P<thread>\d+/\d+), src=(?P<src>[^ ]*) dest=(?P<dest>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)`)
|
|
||||||
)
|
|
||||||
|
|
||||||
type TSParser struct{}
|
|
||||||
|
|
||||||
func (p *TSParser) Parse(line string) (models.LogMessage, error) {
|
|
||||||
newEntry := models.LogMessage{
|
|
||||||
Service: "tixstream",
|
|
||||||
}
|
|
||||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
|
||||||
newEntry.Host = syslogFields.Hostname
|
|
||||||
newEntry.Raw = line
|
|
||||||
newEntry.Type = "service_log"
|
|
||||||
matches := tsServicePattern.FindStringSubmatch(logContent)
|
|
||||||
if len(matches) > 0 {
|
|
||||||
timestampStr := strings.Join(strings.Split(matches[2], " "), "T")
|
|
||||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("unable to parse time", "error", err)
|
|
||||||
}
|
|
||||||
if timestamp.IsZero() {
|
|
||||||
timestamp = syslogFields.SysLogTimestamp
|
|
||||||
}
|
|
||||||
newEntry.LogLevel = strings.TrimSpace(matches[1])
|
|
||||||
newEntry.LogLevel = strings.ReplaceAll(newEntry.LogLevel, "ACE_Message_Block", "")
|
|
||||||
newEntry.Timestamp = timestamp
|
|
||||||
newEntry.LogMessage = strings.TrimSpace(matches[3])
|
|
||||||
} else {
|
|
||||||
newEntry.LogMessage = logContent
|
|
||||||
}
|
|
||||||
var baseInfo models.TSTransferInfo
|
|
||||||
trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
var transferID string
|
|
||||||
if len(trNameMatch) > 0 {
|
|
||||||
transferID = trNameMatch[1]
|
|
||||||
newEntry.LogMessage = trNameMatch[2]
|
|
||||||
split := strings.Fields(trNameMatch[2])
|
|
||||||
switch split[0] {
|
|
||||||
case "in:":
|
|
||||||
baseInfo.Direction = "incoming"
|
|
||||||
case "out:":
|
|
||||||
baseInfo.Direction = "outgoing"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
msg := strings.ReplaceAll(logContent, " ", " ")
|
|
||||||
parts := strings.Fields(msg)
|
|
||||||
|
|
||||||
if len(parts) < 5 {
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
if len(tsDetail) > 0 {
|
|
||||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
|
||||||
baseInfo.Lane = threadInt
|
|
||||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
|
||||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
|
||||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
|
||||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
|
||||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
|
||||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
|
||||||
baseInfo.Buffers = buffersInt
|
|
||||||
baseInfo.FileCount = fileCountInt
|
|
||||||
baseInfo.FileSizeMB = fileSizeFloat
|
|
||||||
baseInfo.ChunkSize = chunkSizeInt
|
|
||||||
baseInfo.Streams = streamsInt
|
|
||||||
baseInfo.TargetDatarate = datarateFloat
|
|
||||||
baseInfo.Protocoll = tsDetail[8]
|
|
||||||
baseInfo.Dest = tsDetail[9]
|
|
||||||
baseInfo.SenderID = tsDetail[10]
|
|
||||||
}
|
|
||||||
tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
if len(tsDetail) > 0 {
|
|
||||||
baseInfo.Target = tsDetail[1]
|
|
||||||
}
|
|
||||||
tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
if len(tsDetail) > 0 {
|
|
||||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
|
||||||
baseInfo.Lane = threadInt
|
|
||||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
|
||||||
baseInfo.Buffers = buffersInt
|
|
||||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
|
||||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
|
||||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
|
||||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
|
||||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
|
||||||
baseInfo.FileCount = fileCountInt
|
|
||||||
baseInfo.FileSizeMB = fileSizeFloat
|
|
||||||
baseInfo.ChunkSize = chunkSizeInt
|
|
||||||
baseInfo.Streams = streamsInt
|
|
||||||
baseInfo.TargetDatarate = datarateFloat
|
|
||||||
baseInfo.Protocoll = tsDetail[8]
|
|
||||||
baseInfo.Src = tsDetail[9]
|
|
||||||
baseInfo.Receiver = tsDetail[10]
|
|
||||||
}
|
|
||||||
tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
|
|
||||||
if len(tsDetail) > 0 {
|
|
||||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
|
||||||
baseInfo.Lane = threadInt
|
|
||||||
baseInfo.Src = tsDetail[2]
|
|
||||||
baseInfo.Dest = tsDetail[3]
|
|
||||||
}
|
|
||||||
if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
|
|
||||||
baseInfo.StartTime = newEntry.Timestamp
|
|
||||||
}
|
|
||||||
if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
|
|
||||||
baseInfo.EndTime = newEntry.Timestamp
|
|
||||||
}
|
|
||||||
if transferID != "" {
|
|
||||||
baseInfo.TransferID = transferID
|
|
||||||
} else {
|
|
||||||
baseInfo.TransferID = "no_transfer_id"
|
|
||||||
}
|
|
||||||
if !baseInfo.StartTime.IsZero() {
|
|
||||||
newEntry.ServiceInformation = baseInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
return newEntry, nil
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue