refactor: use parser package for systemd logs
This commit is contained in:
parent
9aa1b7384d
commit
e468b3a0e3
13 changed files with 897 additions and 275 deletions
|
|
@ -7,6 +7,7 @@ import (
|
|||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"tixel_watch/helpers"
|
||||
"tixel_watch/models"
|
||||
|
||||
"github.com/hpcloud/tail"
|
||||
|
|
@ -175,7 +176,7 @@ func (p *NginxTJMLogParser) parseNginxTJM(entry models.LogMessage) models.LogMes
|
|||
|
||||
if len(parts) > 0 {
|
||||
timestampStr := strings.Trim(parts[0], "[]")
|
||||
timestamp, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("unable to parse time", "error", err)
|
||||
}
|
||||
|
|
|
|||
86
helpers/helpers.go
Normal file
86
helpers/helpers.go
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
package helpers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
var (
|
||||
syslogPattern = regexp.MustCompile(`^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) ([^\s]+) ([^:]+):\s*(.*)$`)
|
||||
)
|
||||
|
||||
func ExtractSyslogHeader(line string) (models.SyslogFields, string) {
|
||||
var syslogFields models.SyslogFields
|
||||
|
||||
matches := syslogPattern.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if len(matches) != 5 {
|
||||
return syslogFields, line
|
||||
}
|
||||
|
||||
sysTime, err := ParseSyslogTimeToRFC3339(matches[1])
|
||||
if err != nil {
|
||||
slog.Error("cant parse sys log time", "error", err)
|
||||
}
|
||||
syslogFields.SysLogTimestamp = sysTime
|
||||
syslogFields.Hostname = matches[2]
|
||||
syslogFields.ProcessInfo = matches[3]
|
||||
|
||||
return syslogFields, matches[4]
|
||||
}
|
||||
|
||||
func GetNamedGroup(match []string, regex *regexp.Regexp, groupName string) string {
|
||||
names := regex.SubexpNames()
|
||||
for i, name := range names {
|
||||
if name == groupName && i < len(match) {
|
||||
return match[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func ParseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
|
||||
if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
|
||||
timeStr += "Z"
|
||||
}
|
||||
return time.Parse(time.RFC3339Nano, timeStr)
|
||||
}
|
||||
|
||||
var deToEnMonth = map[string]string{
|
||||
"Jan": "Jan", "Feb": "Feb", "Mär": "Mar", "Apr": "Apr", "Mai": "May",
|
||||
"Jun": "Jun", "Jul": "Jul", "Aug": "Aug", "Sep": "Sep", "Okt": "Oct",
|
||||
"Nov": "Nov", "Dez": "Dec",
|
||||
}
|
||||
|
||||
func translateMonth(syslogTime string) string {
|
||||
for de, en := range deToEnMonth {
|
||||
if strings.HasPrefix(syslogTime, de) {
|
||||
return strings.Replace(syslogTime, de, en, 1)
|
||||
}
|
||||
}
|
||||
return syslogTime
|
||||
}
|
||||
|
||||
func ParseSyslogTimeToRFC3339(syslogTime string) (time.Time, error) {
|
||||
const syslogLayout = "Jan 02 15:04:05"
|
||||
syslogTime = translateMonth(syslogTime)
|
||||
t, err := time.Parse(syslogLayout, syslogTime)
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("cannot parse syslog time %q: %w", syslogTime, err)
|
||||
}
|
||||
now := time.Now()
|
||||
t = t.AddDate(now.Year(), 0, 0)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func GetHostname() (string, error) {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = "unknown"
|
||||
}
|
||||
return hostname, nil
|
||||
}
|
||||
49
parser/am_parser.go
Normal file
49
parser/am_parser.go
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"tixel_watch/helpers"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
var (
|
||||
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||
)
|
||||
|
||||
type AMParser struct{}
|
||||
|
||||
func (a *AMParser) Parse(line string) (models.LogMessage, error) {
|
||||
newEntry := models.LogMessage{
|
||||
Service: "access-manager",
|
||||
}
|
||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
||||
newEntry.Host = syslogFields.Hostname
|
||||
|
||||
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
||||
if len(matches) != 7 {
|
||||
newEntry.Timestamp = time.Now()
|
||||
newEntry.LogMessage = line
|
||||
return newEntry, nil
|
||||
}
|
||||
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("unable to parse time", "error", err)
|
||||
return newEntry, err
|
||||
}
|
||||
baseInfo := models.AMBaseInfo{
|
||||
ProcessID: matches[3],
|
||||
ThreadID: strings.TrimSpace(matches[4]),
|
||||
LoggerName: matches[5],
|
||||
}
|
||||
newEntry.Timestamp = timestamp
|
||||
newEntry.LogLevel = matches[2]
|
||||
newEntry.LogMessage = matches[6]
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry, nil
|
||||
}
|
||||
28
parser/default_parser.go
Normal file
28
parser/default_parser.go
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
type DefaultParser struct {
|
||||
Service string
|
||||
Tool string
|
||||
}
|
||||
|
||||
func (d *DefaultParser) Parse(line string) (models.LogMessage, error) {
|
||||
msg := models.LogMessage{
|
||||
LogLevel: "unknown",
|
||||
LogMessage: strings.TrimSpace(line),
|
||||
Raw: line,
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
if d.Service != "" {
|
||||
msg.Service = d.Service
|
||||
}
|
||||
if d.Tool != "" {
|
||||
msg.Tool = d.Tool
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
27
parser/factory.go
Normal file
27
parser/factory.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package parser
|
||||
|
||||
func New(serviceName, logType string) (Parser, error) {
|
||||
switch logType {
|
||||
case "custom":
|
||||
switch serviceName {
|
||||
case "tixstream":
|
||||
return &TSParser{}, nil
|
||||
case "transfer-job-manager":
|
||||
return &TJMParser{}, nil
|
||||
case "access-manager":
|
||||
return &arser{}, nil
|
||||
case "tixel-control-center":
|
||||
return &TCCParser{}, nil
|
||||
case "nginx":
|
||||
return &NginxParser{}, nil
|
||||
case "nginx-tjm":
|
||||
return &NginxTJMLogParser{ToolName: serviceName}, nil
|
||||
default:
|
||||
return &DefaultParser{Service: serviceName}, nil
|
||||
}
|
||||
case "json":
|
||||
return &JSONParser{}, nil
|
||||
default:
|
||||
return &DefaultParser{Service: serviceName}, nil
|
||||
}
|
||||
}
|
||||
19
parser/json_parser.go
Normal file
19
parser/json_parser.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
type JSONParser struct{}
|
||||
|
||||
func (j *JSONParser) Parse(line string) (models.LogMessage, error) {
|
||||
var logMsg models.LogMessage
|
||||
|
||||
if err := json.Unmarshal([]byte(line), &logMsg); err == nil {
|
||||
slog.Error("error parsing json line", "error", err)
|
||||
return logMsg, err
|
||||
}
|
||||
return logMsg, nil
|
||||
}
|
||||
57
parser/nginx_parser.go
Normal file
57
parser/nginx_parser.go
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
var (
|
||||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
||||
)
|
||||
|
||||
type NginxParser struct{}
|
||||
|
||||
func (n *NginxParser) Parse(line string) (models.LogMessage, error) {
|
||||
newEntry := models.LogMessage{
|
||||
Service: "nginx",
|
||||
}
|
||||
|
||||
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if len(matches) < 7 {
|
||||
return newEntry, nil
|
||||
}
|
||||
statusCode, err := strconv.ParseInt(matches[5], 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant parse statuscode", "error", err)
|
||||
}
|
||||
bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant parse bytessend", "error", err)
|
||||
}
|
||||
baseInfo := models.NGinXBaseInfo{
|
||||
ClientIP: matches[1],
|
||||
RemoteUser: matches[2],
|
||||
Request: matches[4],
|
||||
StatusCode: int(statusCode),
|
||||
BytesSend: int(bytesSend),
|
||||
}
|
||||
|
||||
if len(matches) > 7 && matches[7] != "" {
|
||||
baseInfo.Referer = matches[7]
|
||||
}
|
||||
if len(matches) > 8 && matches[8] != "" {
|
||||
baseInfo.UserAgent = matches[8]
|
||||
}
|
||||
|
||||
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
|
||||
baseInfo.HTTPMethod = requestParts[0]
|
||||
baseInfo.RequestURI = requestParts[1]
|
||||
baseInfo.HTTPVersion = requestParts[2]
|
||||
}
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry, nil
|
||||
}
|
||||
74
parser/nginx_tjm_parser.go
Normal file
74
parser/nginx_tjm_parser.go
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"tixel_watch/helpers"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
type NginxTJMLogParser struct {
|
||||
ToolName string
|
||||
}
|
||||
|
||||
func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) {
|
||||
entry := models.LogMessage{
|
||||
Type: "log_entry",
|
||||
Tool: p.ToolName,
|
||||
Raw: line,
|
||||
}
|
||||
hostname, err := helpers.GetHostname()
|
||||
if err != nil {
|
||||
return entry, err
|
||||
}
|
||||
entry.Host = hostname
|
||||
entry = p.parseNginxTJM(entry)
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (p *NginxTJMLogParser) parseNginxTJM(entry models.LogMessage) models.LogMessage {
|
||||
newEntry := entry
|
||||
var nginxBase models.NGinXBaseInfo
|
||||
parts := strings.Fields(entry.Raw)
|
||||
if len(parts) < 10 {
|
||||
return newEntry
|
||||
}
|
||||
|
||||
if len(parts) > 0 {
|
||||
timestampStr := strings.Trim(parts[0], "[]")
|
||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("unable to parse time", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timestamp
|
||||
}
|
||||
|
||||
if len(parts) > 2 {
|
||||
nginxBase.ClientIP = parts[2]
|
||||
}
|
||||
|
||||
for i, part := range parts {
|
||||
if strings.HasPrefix(part, "\"") {
|
||||
if i+1 < len(parts) {
|
||||
nginxBase.HTTPMethod = strings.Trim(part, "\"")
|
||||
nginxBase.Route = parts[i+1]
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, part := range parts {
|
||||
if after, ok := strings.CutPrefix(part, "status="); ok {
|
||||
statusCode, err := strconv.ParseInt(after, 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant convert statuscode", "error", err)
|
||||
}
|
||||
nginxBase.StatusCode = int(statusCode)
|
||||
break
|
||||
}
|
||||
}
|
||||
newEntry.ServiceInformation = nginxBase
|
||||
|
||||
return newEntry
|
||||
}
|
||||
11
parser/parser.go
Normal file
11
parser/parser.go
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
type Parser interface {
|
||||
//TODO: Change parsers to return an error as well
|
||||
Parse(line string) (models.LogMessage, error)
|
||||
// Parse(line string) models.LogMessage
|
||||
}
|
||||
49
parser/tcc_parser.go
Normal file
49
parser/tcc_parser.go
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"tixel_watch/helpers"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
var (
|
||||
tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||
)
|
||||
|
||||
type TCCParser struct{}
|
||||
|
||||
func (t *TCCParser) Parse(line string) (models.LogMessage, error) {
|
||||
newEntry := models.LogMessage{
|
||||
Service: "tixel-control-center",
|
||||
}
|
||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
||||
newEntry.Host = syslogFields.Hostname
|
||||
|
||||
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
||||
if len(matches) != 7 {
|
||||
newEntry.Timestamp = time.Now()
|
||||
newEntry.LogMessage = line
|
||||
return newEntry, nil
|
||||
}
|
||||
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("unable to parse time", "error", err)
|
||||
return newEntry, err
|
||||
}
|
||||
baseInfo := models.TCCBaseInfo{
|
||||
ProcessID: matches[3],
|
||||
ThreadID: strings.TrimSpace(matches[4]),
|
||||
LoggerName: matches[5],
|
||||
}
|
||||
newEntry.Timestamp = timestamp
|
||||
newEntry.LogLevel = matches[2]
|
||||
newEntry.LogMessage = matches[6]
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry, nil
|
||||
}
|
||||
91
parser/tjm_parser.go
Normal file
91
parser/tjm_parser.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
"tixel_watch/helpers"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
var (
|
||||
tjmServicePattern = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
|
||||
tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (.*)$`)
|
||||
tjmTransferIDPattern1 = regexp.MustCompile(`(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
|
||||
tjmTransferIDPattern2 = regexp.MustCompile(`(?P<before>.*)(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
|
||||
)
|
||||
|
||||
type TJMParser struct{}
|
||||
|
||||
func (t *TJMParser) Parse(line string) (models.LogMessage, error) {
|
||||
newEntry := models.LogMessage{
|
||||
Service: "transfer-job-manager",
|
||||
}
|
||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
||||
newEntry.Host = syslogFields.Hostname
|
||||
|
||||
msg := strings.TrimSpace(logContent)
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
msg = strings.ReplaceAll(msg, "---", "")
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
parts := strings.Fields(msg)
|
||||
if len(parts) < 4 {
|
||||
newEntry.LogMessage = logContent
|
||||
return newEntry, nil
|
||||
}
|
||||
matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||
var baseInfo models.TJMTransferInfo
|
||||
if len(matches) > 0 {
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("unable to parse time", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timestamp
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[2])
|
||||
newEntry.LogMessage = strings.TrimSpace(matches[8])
|
||||
baseInfo = models.TJMTransferInfo{
|
||||
ProcessID: strings.TrimSpace(matches[3]),
|
||||
CorrelationID: strings.TrimSpace(matches[4]),
|
||||
Username: strings.TrimSpace(matches[5]),
|
||||
ThreadID: strings.TrimSpace(matches[6]),
|
||||
JavaClass: strings.TrimSpace(matches[7]),
|
||||
}
|
||||
} else {
|
||||
newEntry.LogMessage = logContent
|
||||
}
|
||||
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
var transferName string
|
||||
var transferID string
|
||||
if len(trNameMatch) > 0 {
|
||||
transferName = trNameMatch[1]
|
||||
newEntry.LogMessage = trNameMatch[2]
|
||||
if strings.Contains(trNameMatch[1], "-in") {
|
||||
baseInfo.Direction = "incoming"
|
||||
}
|
||||
if strings.Contains(trNameMatch[1], "-out") {
|
||||
baseInfo.Direction = "outgoing"
|
||||
}
|
||||
}
|
||||
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(trIDMatch) > 0 {
|
||||
transferID = trIDMatch[1]
|
||||
}
|
||||
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(trIDMatch) > 0 {
|
||||
transferID = trIDMatch[2]
|
||||
}
|
||||
if transferID != "" {
|
||||
baseInfo.TransferID = transferID
|
||||
} else if transferName != "" {
|
||||
baseInfo.TransferID = transferName
|
||||
} else {
|
||||
baseInfo.TransferID = "no_transfer_id"
|
||||
}
|
||||
if baseInfo.StartTime.IsZero() {
|
||||
baseInfo.StartTime = newEntry.Timestamp
|
||||
}
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry, nil
|
||||
}
|
||||
136
parser/ts_parser.go
Normal file
136
parser/ts_parser.go
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"tixel_watch/helpers"
|
||||
"tixel_watch/models"
|
||||
)
|
||||
|
||||
var (
|
||||
tsServicePattern = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
|
||||
tsTransferIDPattern = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
|
||||
tsDetailPattern1 = regexp.MustCompile(`in: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<dest>\S+) sender-id=(?P<sender_id>\S+)`)
|
||||
tsDetailPattern2 = regexp.MustCompile(`out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s`)
|
||||
tsDetailPattern3 = regexp.MustCompile(`out: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<src>\S+) receiver=(?P<receiver>\S+)`)
|
||||
tsDetailPattern4 = regexp.MustCompile(`out: Start transfer (?P<thread>\d+/\d+), src=(?P<src>[^ ]*) dest=(?P<dest>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)`)
|
||||
)
|
||||
|
||||
type TSParser struct{}
|
||||
|
||||
func (p *TSParser) Parse(line string) (models.LogMessage, error) {
|
||||
newEntry := models.LogMessage{
|
||||
Service: "tixstream",
|
||||
}
|
||||
syslogFields, logContent := helpers.ExtractSyslogHeader(line)
|
||||
newEntry.Host = syslogFields.Hostname
|
||||
newEntry.Raw = line
|
||||
newEntry.Type = "service_log"
|
||||
matches := tsServicePattern.FindStringSubmatch(logContent)
|
||||
if len(matches) > 0 {
|
||||
timestampStr := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("unable to parse time", "error", err)
|
||||
}
|
||||
if timestamp.IsZero() {
|
||||
timestamp = syslogFields.SysLogTimestamp
|
||||
}
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
newEntry.LogLevel = strings.ReplaceAll(newEntry.LogLevel, "ACE_Message_Block", "")
|
||||
newEntry.Timestamp = timestamp
|
||||
newEntry.LogMessage = strings.TrimSpace(matches[3])
|
||||
} else {
|
||||
newEntry.LogMessage = logContent
|
||||
}
|
||||
var baseInfo models.TSTransferInfo
|
||||
trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
var transferID string
|
||||
if len(trNameMatch) > 0 {
|
||||
transferID = trNameMatch[1]
|
||||
newEntry.LogMessage = trNameMatch[2]
|
||||
split := strings.Fields(trNameMatch[2])
|
||||
switch split[0] {
|
||||
case "in:":
|
||||
baseInfo.Direction = "incoming"
|
||||
case "out:":
|
||||
baseInfo.Direction = "outgoing"
|
||||
}
|
||||
}
|
||||
|
||||
msg := strings.ReplaceAll(logContent, " ", " ")
|
||||
parts := strings.Fields(msg)
|
||||
|
||||
if len(parts) < 5 {
|
||||
return newEntry, nil
|
||||
}
|
||||
tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
baseInfo.Lane = threadInt
|
||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
baseInfo.Buffers = buffersInt
|
||||
baseInfo.FileCount = fileCountInt
|
||||
baseInfo.FileSizeMB = fileSizeFloat
|
||||
baseInfo.ChunkSize = chunkSizeInt
|
||||
baseInfo.Streams = streamsInt
|
||||
baseInfo.TargetDatarate = datarateFloat
|
||||
baseInfo.Protocoll = tsDetail[8]
|
||||
baseInfo.Dest = tsDetail[9]
|
||||
baseInfo.SenderID = tsDetail[10]
|
||||
}
|
||||
tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
baseInfo.Target = tsDetail[1]
|
||||
}
|
||||
tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
baseInfo.Lane = threadInt
|
||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
baseInfo.Buffers = buffersInt
|
||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
baseInfo.FileCount = fileCountInt
|
||||
baseInfo.FileSizeMB = fileSizeFloat
|
||||
baseInfo.ChunkSize = chunkSizeInt
|
||||
baseInfo.Streams = streamsInt
|
||||
baseInfo.TargetDatarate = datarateFloat
|
||||
baseInfo.Protocoll = tsDetail[8]
|
||||
baseInfo.Src = tsDetail[9]
|
||||
baseInfo.Receiver = tsDetail[10]
|
||||
}
|
||||
tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Src = tsDetail[2]
|
||||
baseInfo.Dest = tsDetail[3]
|
||||
}
|
||||
if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
|
||||
baseInfo.StartTime = newEntry.Timestamp
|
||||
}
|
||||
if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
|
||||
baseInfo.EndTime = newEntry.Timestamp
|
||||
}
|
||||
if transferID != "" {
|
||||
baseInfo.TransferID = transferID
|
||||
} else {
|
||||
baseInfo.TransferID = "no_transfer_id"
|
||||
}
|
||||
if !baseInfo.StartTime.IsZero() {
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
}
|
||||
|
||||
return newEntry, nil
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
"tixel_watch/models"
|
||||
"tixel_watch/parser"
|
||||
)
|
||||
|
||||
type ServiceMonitor struct {
|
||||
|
|
@ -207,20 +208,13 @@ func (jep *JournalEntryParser) extractSystemdFields(journalData map[string]any,
|
|||
}
|
||||
|
||||
func (jep *JournalEntryParser) parseServiceSpecific(entry models.LogMessage) models.LogMessage {
|
||||
switch jep.serviceName {
|
||||
case "tixstream":
|
||||
return parseTixstreamService(entry)
|
||||
case "transfer-job-manager":
|
||||
return parseTJMService(entry)
|
||||
case "nginx":
|
||||
return parseNginxService(entry)
|
||||
case "access-manager":
|
||||
return parseAMService(entry)
|
||||
case "tixel-control-center":
|
||||
return parseTCCService(entry)
|
||||
default:
|
||||
logParser, err := parser.New(jep.serviceName, "custom")
|
||||
if err != nil {
|
||||
slog.Error("cannot get service specific parser")
|
||||
return entry
|
||||
}
|
||||
entry, err = logParser.Parse(entry.LogMessage)
|
||||
return entry
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -239,283 +233,283 @@ var (
|
|||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
||||
)
|
||||
|
||||
func parseTixstreamService(entry models.LogMessage) models.LogMessage {
|
||||
newEntry := entry
|
||||
var baseInfo models.TSTransferInfo
|
||||
// func parseTixstreamService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// var baseInfo models.TSTransferInfo
|
||||
|
||||
matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(matches) > 0 {
|
||||
timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
newEntry.LogMessage = strings.TrimSpace(matches[3])
|
||||
}
|
||||
trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
var transferID string
|
||||
if len(trNameMatch) > 0 {
|
||||
transferID = trNameMatch[1]
|
||||
newEntry.LogMessage = trNameMatch[2]
|
||||
split := strings.Fields(trNameMatch[2])
|
||||
switch split[0] {
|
||||
case "in:":
|
||||
baseInfo.Direction = "incoming"
|
||||
case "out:":
|
||||
baseInfo.Direction = "outgoing"
|
||||
}
|
||||
}
|
||||
// matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(matches) > 0 {
|
||||
// timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
// newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// newEntry.LogMessage = strings.TrimSpace(matches[3])
|
||||
// }
|
||||
// trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
// var transferID string
|
||||
// if len(trNameMatch) > 0 {
|
||||
// transferID = trNameMatch[1]
|
||||
// newEntry.LogMessage = trNameMatch[2]
|
||||
// split := strings.Fields(trNameMatch[2])
|
||||
// switch split[0] {
|
||||
// case "in:":
|
||||
// baseInfo.Direction = "incoming"
|
||||
// case "out:":
|
||||
// baseInfo.Direction = "outgoing"
|
||||
// }
|
||||
// }
|
||||
|
||||
msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ")
|
||||
parts := strings.Fields(msg)
|
||||
// msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ")
|
||||
// parts := strings.Fields(msg)
|
||||
|
||||
if len(parts) < 5 {
|
||||
return newEntry
|
||||
}
|
||||
tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Buffers = buffersInt
|
||||
baseInfo.FileCount = fileCountInt
|
||||
baseInfo.FileSizeMB = fileSizeFloat
|
||||
baseInfo.ChunkSize = chunkSizeInt
|
||||
baseInfo.Streams = streamsInt
|
||||
baseInfo.TargetDatarate = datarateFloat
|
||||
baseInfo.Protocoll = tsDetail[8]
|
||||
baseInfo.Dest = tsDetail[9]
|
||||
baseInfo.SenderID = tsDetail[10]
|
||||
}
|
||||
tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
baseInfo.Target = tsDetail[1]
|
||||
}
|
||||
tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Buffers = buffersInt
|
||||
baseInfo.FileCount = fileCountInt
|
||||
baseInfo.FileSizeMB = fileSizeFloat
|
||||
baseInfo.ChunkSize = chunkSizeInt
|
||||
baseInfo.Streams = streamsInt
|
||||
baseInfo.TargetDatarate = datarateFloat
|
||||
baseInfo.Protocoll = tsDetail[8]
|
||||
baseInfo.Src = tsDetail[9]
|
||||
baseInfo.Receiver = tsDetail[10]
|
||||
}
|
||||
tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Src = tsDetail[2]
|
||||
baseInfo.Dest = tsDetail[3]
|
||||
}
|
||||
if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
|
||||
baseInfo.StartTime = newEntry.Timestamp
|
||||
} else {
|
||||
baseInfo.StartTime = time.Now()
|
||||
}
|
||||
if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
|
||||
baseInfo.EndTime = newEntry.Timestamp
|
||||
} else {
|
||||
baseInfo.EndTime = baseInfo.StartTime
|
||||
}
|
||||
if transferID != "" {
|
||||
baseInfo.TransferID = transferID
|
||||
} else {
|
||||
baseInfo.TransferID = "no_transfer_id"
|
||||
}
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
return newEntry
|
||||
}
|
||||
// if len(parts) < 5 {
|
||||
// return newEntry
|
||||
// }
|
||||
// tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
// buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
// fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
// chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
// streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
// datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
// fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
// baseInfo.Lane = threadInt
|
||||
// baseInfo.Buffers = buffersInt
|
||||
// baseInfo.FileCount = fileCountInt
|
||||
// baseInfo.FileSizeMB = fileSizeFloat
|
||||
// baseInfo.ChunkSize = chunkSizeInt
|
||||
// baseInfo.Streams = streamsInt
|
||||
// baseInfo.TargetDatarate = datarateFloat
|
||||
// baseInfo.Protocoll = tsDetail[8]
|
||||
// baseInfo.Dest = tsDetail[9]
|
||||
// baseInfo.SenderID = tsDetail[10]
|
||||
// }
|
||||
// tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// baseInfo.Target = tsDetail[1]
|
||||
// }
|
||||
// tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
// buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
// fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
// fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
// chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
// streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
// datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
// baseInfo.Lane = threadInt
|
||||
// baseInfo.Buffers = buffersInt
|
||||
// baseInfo.FileCount = fileCountInt
|
||||
// baseInfo.FileSizeMB = fileSizeFloat
|
||||
// baseInfo.ChunkSize = chunkSizeInt
|
||||
// baseInfo.Streams = streamsInt
|
||||
// baseInfo.TargetDatarate = datarateFloat
|
||||
// baseInfo.Protocoll = tsDetail[8]
|
||||
// baseInfo.Src = tsDetail[9]
|
||||
// baseInfo.Receiver = tsDetail[10]
|
||||
// }
|
||||
// tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
// baseInfo.Lane = threadInt
|
||||
// baseInfo.Src = tsDetail[2]
|
||||
// baseInfo.Dest = tsDetail[3]
|
||||
// }
|
||||
// if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
|
||||
// baseInfo.StartTime = newEntry.Timestamp
|
||||
// } else {
|
||||
// baseInfo.StartTime = time.Now()
|
||||
// }
|
||||
// if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
|
||||
// baseInfo.EndTime = newEntry.Timestamp
|
||||
// } else {
|
||||
// baseInfo.EndTime = baseInfo.StartTime
|
||||
// }
|
||||
// if transferID != "" {
|
||||
// baseInfo.TransferID = transferID
|
||||
// } else {
|
||||
// baseInfo.TransferID = "no_transfer_id"
|
||||
// }
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseTJMService(entry models.LogMessage) models.LogMessage {
|
||||
newEntry := entry
|
||||
var baseInfo models.TJMTransferInfo
|
||||
// func parseTJMService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// var baseInfo models.TJMTransferInfo
|
||||
|
||||
logContent := entry.LogMessage
|
||||
msg := strings.TrimSpace(logContent)
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
msg = strings.ReplaceAll(msg, "---", "")
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
parts := strings.Fields(msg)
|
||||
if len(parts) < 4 {
|
||||
return newEntry
|
||||
}
|
||||
matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||
if len(matches) > 0 {
|
||||
timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[2])
|
||||
newEntry.LogMessage = strings.TrimSpace(matches[8])
|
||||
baseInfo = models.TJMTransferInfo{
|
||||
ProcessID: strings.TrimSpace(matches[3]),
|
||||
CorrelationID: strings.TrimSpace(matches[4]),
|
||||
Username: strings.TrimSpace(matches[5]),
|
||||
ThreadID: strings.TrimSpace(matches[6]),
|
||||
JavaClass: strings.TrimSpace(matches[7]),
|
||||
}
|
||||
} else {
|
||||
newEntry.LogMessage = logContent
|
||||
}
|
||||
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
var transferName string
|
||||
var transferID string
|
||||
if len(trNameMatch) > 0 {
|
||||
transferName = trNameMatch[1]
|
||||
newEntry.LogMessage = trNameMatch[2]
|
||||
if strings.Contains(trNameMatch[1], "-in") {
|
||||
baseInfo.Direction = "incoming"
|
||||
}
|
||||
if strings.Contains(trNameMatch[1], "-out") {
|
||||
baseInfo.Direction = "outgoing"
|
||||
}
|
||||
}
|
||||
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(trIDMatch) > 0 {
|
||||
transferID = trIDMatch[1]
|
||||
}
|
||||
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(trIDMatch) > 0 {
|
||||
transferID = trIDMatch[2]
|
||||
}
|
||||
if transferID != "" {
|
||||
baseInfo.TransferID = transferID
|
||||
} else if transferName != "" {
|
||||
baseInfo.TransferID = transferName
|
||||
} else {
|
||||
baseInfo.TransferID = "no_transfer_id"
|
||||
}
|
||||
baseInfo.StartTime = newEntry.Timestamp
|
||||
baseInfo.StartTime = newEntry.Timestamp
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
// logContent := entry.LogMessage
|
||||
// msg := strings.TrimSpace(logContent)
|
||||
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||
// msg = strings.ReplaceAll(msg, "---", "")
|
||||
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||
// parts := strings.Fields(msg)
|
||||
// if len(parts) < 4 {
|
||||
// return newEntry
|
||||
// }
|
||||
// matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||
// if len(matches) > 0 {
|
||||
// timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
// newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// newEntry.LogLevel = strings.TrimSpace(matches[2])
|
||||
// newEntry.LogMessage = strings.TrimSpace(matches[8])
|
||||
// baseInfo = models.TJMTransferInfo{
|
||||
// ProcessID: strings.TrimSpace(matches[3]),
|
||||
// CorrelationID: strings.TrimSpace(matches[4]),
|
||||
// Username: strings.TrimSpace(matches[5]),
|
||||
// ThreadID: strings.TrimSpace(matches[6]),
|
||||
// JavaClass: strings.TrimSpace(matches[7]),
|
||||
// }
|
||||
// } else {
|
||||
// newEntry.LogMessage = logContent
|
||||
// }
|
||||
// trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
// var transferName string
|
||||
// var transferID string
|
||||
// if len(trNameMatch) > 0 {
|
||||
// transferName = trNameMatch[1]
|
||||
// newEntry.LogMessage = trNameMatch[2]
|
||||
// if strings.Contains(trNameMatch[1], "-in") {
|
||||
// baseInfo.Direction = "incoming"
|
||||
// }
|
||||
// if strings.Contains(trNameMatch[1], "-out") {
|
||||
// baseInfo.Direction = "outgoing"
|
||||
// }
|
||||
// }
|
||||
// trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(trIDMatch) > 0 {
|
||||
// transferID = trIDMatch[1]
|
||||
// }
|
||||
// trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(trIDMatch) > 0 {
|
||||
// transferID = trIDMatch[2]
|
||||
// }
|
||||
// if transferID != "" {
|
||||
// baseInfo.TransferID = transferID
|
||||
// } else if transferName != "" {
|
||||
// baseInfo.TransferID = transferName
|
||||
// } else {
|
||||
// baseInfo.TransferID = "no_transfer_id"
|
||||
// }
|
||||
// baseInfo.StartTime = newEntry.Timestamp
|
||||
// baseInfo.StartTime = newEntry.Timestamp
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseAMService(entry models.LogMessage) models.LogMessage {
|
||||
newEntry := entry
|
||||
logContent := newEntry.LogMessage
|
||||
// func parseAMService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// logContent := newEntry.LogMessage
|
||||
|
||||
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
||||
if len(matches) != 7 {
|
||||
return newEntry
|
||||
}
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
baseInfo := models.AMBaseInfo{
|
||||
ProcessID: matches[3],
|
||||
ThreadID: strings.TrimSpace(matches[4]),
|
||||
LoggerName: matches[5],
|
||||
}
|
||||
newEntry.LogLevel = matches[2]
|
||||
newEntry.LogMessage = matches[6]
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
// matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
||||
// if len(matches) != 7 {
|
||||
// return newEntry
|
||||
// }
|
||||
// timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// baseInfo := models.AMBaseInfo{
|
||||
// ProcessID: matches[3],
|
||||
// ThreadID: strings.TrimSpace(matches[4]),
|
||||
// LoggerName: matches[5],
|
||||
// }
|
||||
// newEntry.LogLevel = matches[2]
|
||||
// newEntry.LogMessage = matches[6]
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseTCCService(entry models.LogMessage) models.LogMessage {
|
||||
newEntry := entry
|
||||
logContent := newEntry.LogMessage
|
||||
// func parseTCCService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// logContent := newEntry.LogMessage
|
||||
|
||||
matches := tccServicePattern.FindStringSubmatch(logContent)
|
||||
if len(matches) != 7 {
|
||||
return newEntry
|
||||
}
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
baseInfo := models.TCCBaseInfo{
|
||||
ProcessID: matches[3],
|
||||
ThreadID: strings.TrimSpace(matches[4]),
|
||||
LoggerName: matches[5],
|
||||
}
|
||||
newEntry.LogLevel = matches[2]
|
||||
newEntry.LogMessage = matches[6]
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
// matches := tccServicePattern.FindStringSubmatch(logContent)
|
||||
// if len(matches) != 7 {
|
||||
// return newEntry
|
||||
// }
|
||||
// timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// baseInfo := models.TCCBaseInfo{
|
||||
// ProcessID: matches[3],
|
||||
// ThreadID: strings.TrimSpace(matches[4]),
|
||||
// LoggerName: matches[5],
|
||||
// }
|
||||
// newEntry.LogLevel = matches[2]
|
||||
// newEntry.LogMessage = matches[6]
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseNginxService(entry models.LogMessage) models.LogMessage {
|
||||
newEntry := entry
|
||||
// func parseNginxService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
|
||||
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
|
||||
if len(matches) < 7 {
|
||||
return newEntry
|
||||
}
|
||||
statusCode, err := strconv.ParseInt(matches[5], 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant parse statuscode", "error", err)
|
||||
}
|
||||
bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant parse bytessend", "error", err)
|
||||
}
|
||||
baseInfo := models.NGinXBaseInfo{
|
||||
ClientIP: matches[1],
|
||||
RemoteUser: matches[2],
|
||||
Request: matches[4],
|
||||
StatusCode: int(statusCode),
|
||||
BytesSend: int(bytesSend),
|
||||
}
|
||||
// matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
|
||||
// if len(matches) < 7 {
|
||||
// return newEntry
|
||||
// }
|
||||
// statusCode, err := strconv.ParseInt(matches[5], 10, 64)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse statuscode", "error", err)
|
||||
// }
|
||||
// bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse bytessend", "error", err)
|
||||
// }
|
||||
// baseInfo := models.NGinXBaseInfo{
|
||||
// ClientIP: matches[1],
|
||||
// RemoteUser: matches[2],
|
||||
// Request: matches[4],
|
||||
// StatusCode: int(statusCode),
|
||||
// BytesSend: int(bytesSend),
|
||||
// }
|
||||
|
||||
if len(matches) > 7 && matches[7] != "" {
|
||||
baseInfo.Referer = matches[7]
|
||||
}
|
||||
if len(matches) > 8 && matches[8] != "" {
|
||||
baseInfo.UserAgent = matches[8]
|
||||
}
|
||||
// if len(matches) > 7 && matches[7] != "" {
|
||||
// baseInfo.Referer = matches[7]
|
||||
// }
|
||||
// if len(matches) > 8 && matches[8] != "" {
|
||||
// baseInfo.UserAgent = matches[8]
|
||||
// }
|
||||
|
||||
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
|
||||
baseInfo.HTTPMethod = requestParts[0]
|
||||
baseInfo.RequestURI = requestParts[1]
|
||||
baseInfo.HTTPVersion = requestParts[2]
|
||||
}
|
||||
newEntry.ServiceInformation = baseInfo
|
||||
// if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
|
||||
// baseInfo.HTTPMethod = requestParts[0]
|
||||
// baseInfo.RequestURI = requestParts[1]
|
||||
// baseInfo.HTTPVersion = requestParts[2]
|
||||
// }
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
|
||||
if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
|
||||
timeStr += "Z"
|
||||
}
|
||||
return time.Parse(time.RFC3339Nano, timeStr)
|
||||
}
|
||||
// func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
|
||||
// if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
|
||||
// timeStr += "Z"
|
||||
// }
|
||||
// return time.Parse(time.RFC3339Nano, timeStr)
|
||||
// }
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue