feat: implement new generic parser and improve production readyness
This commit is contained in:
parent
8364218234
commit
0830b403e0
34 changed files with 1715 additions and 2114 deletions
|
|
@ -11,17 +11,19 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"tixel_watch/models"
|
||||
"tixel_watch/parser"
|
||||
"watch-tool/models"
|
||||
"watch-tool/parser"
|
||||
)
|
||||
|
||||
type ServiceMonitor struct {
|
||||
config ServiceConfig
|
||||
config ServiceConfig
|
||||
hostname string
|
||||
}
|
||||
|
||||
func NewServiceMonitor(config ServiceConfig) *ServiceMonitor {
|
||||
func NewServiceMonitor(config ServiceConfig, hostname string) *ServiceMonitor {
|
||||
return &ServiceMonitor{
|
||||
config: config,
|
||||
config: config,
|
||||
hostname: hostname,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -50,7 +52,7 @@ func (sm *ServiceMonitor) Start(ctx context.Context, out chan<- models.LogMessag
|
|||
}
|
||||
}()
|
||||
|
||||
parser := NewJournalEntryParser(sm.config.Name, sm.config.Service)
|
||||
parser := NewJournalEntryParser(sm.config.Name, sm.config.Service, sm.hostname)
|
||||
|
||||
for scanner.Scan() {
|
||||
select {
|
||||
|
|
@ -109,12 +111,14 @@ func (sm *ServiceMonitor) buildJournalctlArgs() []string {
|
|||
type JournalEntryParser struct {
|
||||
serviceName string
|
||||
unitName string
|
||||
hostname string
|
||||
}
|
||||
|
||||
func NewJournalEntryParser(serviceName, unitName string) *JournalEntryParser {
|
||||
func NewJournalEntryParser(serviceName, unitName, hostname string) *JournalEntryParser {
|
||||
return &JournalEntryParser{
|
||||
serviceName: serviceName,
|
||||
unitName: unitName,
|
||||
hostname: hostname,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -124,7 +128,7 @@ func (jep *JournalEntryParser) Parse(jsonLine string) (models.LogMessage, error)
|
|||
return models.LogMessage{}, fmt.Errorf("JSON unmarshal error: %w", err)
|
||||
}
|
||||
|
||||
entry := models.NewLogMessage("service_log", hostname)
|
||||
entry := models.NewLogMessage("service_log", jep.hostname)
|
||||
entry.Service = jep.serviceName
|
||||
entry.Unit = jep.unitName
|
||||
|
||||
|
|
@ -211,7 +215,7 @@ func (jep *JournalEntryParser) extractSystemdFields(journalData map[string]any,
|
|||
}
|
||||
|
||||
func (jep *JournalEntryParser) parseServiceSpecific(entry models.LogMessage) models.LogMessage {
|
||||
logParser, err := parser.New(jep.serviceName, "custom")
|
||||
logParser, err := parser.New(jep.serviceName, "custom", jep.hostname)
|
||||
if err != nil {
|
||||
slog.Error("cannot get service specific parser")
|
||||
return entry
|
||||
|
|
@ -235,284 +239,3 @@ var (
|
|||
tsDetailPattern4 = regexp.MustCompile(`out: Start transfer (?P<thread>\d+/\d+), src=(?P<src>[^ ]*) dest=(?P<dest>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)`)
|
||||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
||||
)
|
||||
|
||||
// func parseTixstreamService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// var baseInfo models.TSTransferInfo
|
||||
|
||||
// matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(matches) > 0 {
|
||||
// timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
// newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// newEntry.LogMessage = strings.TrimSpace(matches[3])
|
||||
// }
|
||||
// trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
// var transferID string
|
||||
// if len(trNameMatch) > 0 {
|
||||
// transferID = trNameMatch[1]
|
||||
// newEntry.LogMessage = trNameMatch[2]
|
||||
// split := strings.Fields(trNameMatch[2])
|
||||
// switch split[0] {
|
||||
// case "in:":
|
||||
// baseInfo.Direction = "incoming"
|
||||
// case "out:":
|
||||
// baseInfo.Direction = "outgoing"
|
||||
// }
|
||||
// }
|
||||
|
||||
// msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ")
|
||||
// parts := strings.Fields(msg)
|
||||
|
||||
// if len(parts) < 5 {
|
||||
// return newEntry
|
||||
// }
|
||||
// tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
// buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
// fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
// chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
// streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
// datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
// fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
// baseInfo.Lane = threadInt
|
||||
// baseInfo.Buffers = buffersInt
|
||||
// baseInfo.FileCount = fileCountInt
|
||||
// baseInfo.FileSizeMB = fileSizeFloat
|
||||
// baseInfo.ChunkSize = chunkSizeInt
|
||||
// baseInfo.Streams = streamsInt
|
||||
// baseInfo.TargetDatarate = datarateFloat
|
||||
// baseInfo.Protocoll = tsDetail[8]
|
||||
// baseInfo.Dest = tsDetail[9]
|
||||
// baseInfo.SenderID = tsDetail[10]
|
||||
// }
|
||||
// tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// baseInfo.Target = tsDetail[1]
|
||||
// }
|
||||
// tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
// buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
// fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
// fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
// chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
// streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
// datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
// baseInfo.Lane = threadInt
|
||||
// baseInfo.Buffers = buffersInt
|
||||
// baseInfo.FileCount = fileCountInt
|
||||
// baseInfo.FileSizeMB = fileSizeFloat
|
||||
// baseInfo.ChunkSize = chunkSizeInt
|
||||
// baseInfo.Streams = streamsInt
|
||||
// baseInfo.TargetDatarate = datarateFloat
|
||||
// baseInfo.Protocoll = tsDetail[8]
|
||||
// baseInfo.Src = tsDetail[9]
|
||||
// baseInfo.Receiver = tsDetail[10]
|
||||
// }
|
||||
// tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(tsDetail) > 0 {
|
||||
// threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
// baseInfo.Lane = threadInt
|
||||
// baseInfo.Src = tsDetail[2]
|
||||
// baseInfo.Dest = tsDetail[3]
|
||||
// }
|
||||
// if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
|
||||
// baseInfo.StartTime = newEntry.Timestamp
|
||||
// } else {
|
||||
// baseInfo.StartTime = time.Now()
|
||||
// }
|
||||
// if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
|
||||
// baseInfo.EndTime = newEntry.Timestamp
|
||||
// } else {
|
||||
// baseInfo.EndTime = baseInfo.StartTime
|
||||
// }
|
||||
// if transferID != "" {
|
||||
// baseInfo.TransferID = transferID
|
||||
// } else {
|
||||
// baseInfo.TransferID = "no_transfer_id"
|
||||
// }
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
// func parseTJMService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// var baseInfo models.TJMTransferInfo
|
||||
|
||||
// logContent := entry.LogMessage
|
||||
// msg := strings.TrimSpace(logContent)
|
||||
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||
// msg = strings.ReplaceAll(msg, "---", "")
|
||||
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||
// parts := strings.Fields(msg)
|
||||
// if len(parts) < 4 {
|
||||
// return newEntry
|
||||
// }
|
||||
// matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||
// if len(matches) > 0 {
|
||||
// timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
// newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// newEntry.LogLevel = strings.TrimSpace(matches[2])
|
||||
// newEntry.LogMessage = strings.TrimSpace(matches[8])
|
||||
// baseInfo = models.TJMTransferInfo{
|
||||
// ProcessID: strings.TrimSpace(matches[3]),
|
||||
// CorrelationID: strings.TrimSpace(matches[4]),
|
||||
// Username: strings.TrimSpace(matches[5]),
|
||||
// ThreadID: strings.TrimSpace(matches[6]),
|
||||
// JavaClass: strings.TrimSpace(matches[7]),
|
||||
// }
|
||||
// } else {
|
||||
// newEntry.LogMessage = logContent
|
||||
// }
|
||||
// trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
// var transferName string
|
||||
// var transferID string
|
||||
// if len(trNameMatch) > 0 {
|
||||
// transferName = trNameMatch[1]
|
||||
// newEntry.LogMessage = trNameMatch[2]
|
||||
// if strings.Contains(trNameMatch[1], "-in") {
|
||||
// baseInfo.Direction = "incoming"
|
||||
// }
|
||||
// if strings.Contains(trNameMatch[1], "-out") {
|
||||
// baseInfo.Direction = "outgoing"
|
||||
// }
|
||||
// }
|
||||
// trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(trIDMatch) > 0 {
|
||||
// transferID = trIDMatch[1]
|
||||
// }
|
||||
// trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
// if len(trIDMatch) > 0 {
|
||||
// transferID = trIDMatch[2]
|
||||
// }
|
||||
// if transferID != "" {
|
||||
// baseInfo.TransferID = transferID
|
||||
// } else if transferName != "" {
|
||||
// baseInfo.TransferID = transferName
|
||||
// } else {
|
||||
// baseInfo.TransferID = "no_transfer_id"
|
||||
// }
|
||||
// baseInfo.StartTime = newEntry.Timestamp
|
||||
// baseInfo.StartTime = newEntry.Timestamp
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
// func parseAMService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// logContent := newEntry.LogMessage
|
||||
|
||||
// matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
||||
// if len(matches) != 7 {
|
||||
// return newEntry
|
||||
// }
|
||||
// timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// baseInfo := models.AMBaseInfo{
|
||||
// ProcessID: matches[3],
|
||||
// ThreadID: strings.TrimSpace(matches[4]),
|
||||
// LoggerName: matches[5],
|
||||
// }
|
||||
// newEntry.LogLevel = matches[2]
|
||||
// newEntry.LogMessage = matches[6]
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
// func parseTCCService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
// logContent := newEntry.LogMessage
|
||||
|
||||
// matches := tccServicePattern.FindStringSubmatch(logContent)
|
||||
// if len(matches) != 7 {
|
||||
// return newEntry
|
||||
// }
|
||||
// timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
// if newEntry.Timestamp.IsZero() {
|
||||
// timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse time string", "error", err)
|
||||
// }
|
||||
// newEntry.Timestamp = timeParsed
|
||||
// }
|
||||
// baseInfo := models.TCCBaseInfo{
|
||||
// ProcessID: matches[3],
|
||||
// ThreadID: strings.TrimSpace(matches[4]),
|
||||
// LoggerName: matches[5],
|
||||
// }
|
||||
// newEntry.LogLevel = matches[2]
|
||||
// newEntry.LogMessage = matches[6]
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
// func parseNginxService(entry models.LogMessage) models.LogMessage {
|
||||
// newEntry := entry
|
||||
|
||||
// matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
|
||||
// if len(matches) < 7 {
|
||||
// return newEntry
|
||||
// }
|
||||
// statusCode, err := strconv.ParseInt(matches[5], 10, 64)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse statuscode", "error", err)
|
||||
// }
|
||||
// bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
|
||||
// if err != nil {
|
||||
// slog.Error("cant parse bytessend", "error", err)
|
||||
// }
|
||||
// baseInfo := models.NGinXBaseInfo{
|
||||
// ClientIP: matches[1],
|
||||
// RemoteUser: matches[2],
|
||||
// Request: matches[4],
|
||||
// StatusCode: int(statusCode),
|
||||
// BytesSend: int(bytesSend),
|
||||
// }
|
||||
|
||||
// if len(matches) > 7 && matches[7] != "" {
|
||||
// baseInfo.Referer = matches[7]
|
||||
// }
|
||||
// if len(matches) > 8 && matches[8] != "" {
|
||||
// baseInfo.UserAgent = matches[8]
|
||||
// }
|
||||
|
||||
// if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
|
||||
// baseInfo.HTTPMethod = requestParts[0]
|
||||
// baseInfo.RequestURI = requestParts[1]
|
||||
// baseInfo.HTTPVersion = requestParts[2]
|
||||
// }
|
||||
// newEntry.ServiceInformation = baseInfo
|
||||
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
// func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
|
||||
// if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
|
||||
// timeStr += "Z"
|
||||
// }
|
||||
// return time.Parse(time.RFC3339Nano, timeStr)
|
||||
// }
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue