feat(service-monitor): update search pattern for better results
This commit is contained in:
parent
81d32a638f
commit
a8eb0ad726
1 changed files with 225 additions and 63 deletions
|
|
@ -212,56 +212,106 @@ func (jep *JournalEntryParser) parseServiceSpecific(entry LogEntry) LogEntry {
|
||||||
return parseNginxService(entry)
|
return parseNginxService(entry)
|
||||||
case "access-manager":
|
case "access-manager":
|
||||||
return parseAMService(entry)
|
return parseAMService(entry)
|
||||||
|
case "tixel-control-center":
|
||||||
|
return parseTCCService(entry)
|
||||||
default:
|
default:
|
||||||
return entry
|
return entry
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||||
|
tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||||
|
tjmServicePattern = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
|
||||||
|
tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-[A-Za-z]+-(?:in|out)) ?: (.*)$`)
|
||||||
|
tsServicePattern = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
|
||||||
|
tsTransferIDPattern = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
|
||||||
|
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
||||||
|
tjmInnerLogPattern = regexp.MustCompile(`^(?P<transfername>[^ ]+) : (?P<javaclass>\w+)\.(?P<methode>\w+): started transfer session (?P<transferid>[a-f0-9\-]+) on (?P<localaddr>[\d\.]+:\d+) with target address (?P<targetaddr>[a-zA-Z0-9\.\-]+:\d+)`)
|
||||||
|
)
|
||||||
|
|
||||||
func parseTixstreamService(entry LogEntry) LogEntry {
|
func parseTixstreamService(entry LogEntry) LogEntry {
|
||||||
newEntry := entry
|
newEntry := entry
|
||||||
msg := strings.ReplaceAll(entry.Message, " ", " ")
|
fields := make(map[string]any)
|
||||||
|
matches := tsServicePattern.FindStringSubmatch(newEntry.Message)
|
||||||
|
if len(matches) > 0 {
|
||||||
|
timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||||
|
fields["log_level"] = strings.TrimSpace(matches[1])
|
||||||
|
fields["message_timestamp"] = timestamp
|
||||||
|
fields["log_message"] = strings.TrimSpace(matches[3])
|
||||||
|
} else {
|
||||||
|
fields["log_message"] = newEntry.Message
|
||||||
|
}
|
||||||
|
trNameMatch := tsTransferIDPattern.FindStringSubmatch(fields["log_message"].(string))
|
||||||
|
if len(trNameMatch) > 0 {
|
||||||
|
fields["transfer_id"] = trNameMatch[1]
|
||||||
|
fields["log_message"] = trNameMatch[2]
|
||||||
|
split := strings.Fields(trNameMatch[2])
|
||||||
|
switch split[0] {
|
||||||
|
case "in:":
|
||||||
|
fields["transfer_direction"] = "incoming"
|
||||||
|
case "out:":
|
||||||
|
fields["transfer_direction"] = "outgoing"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := strings.ReplaceAll(newEntry.Message, " ", " ")
|
||||||
parts := strings.Fields(msg)
|
parts := strings.Fields(msg)
|
||||||
|
|
||||||
if len(parts) < 5 {
|
if len(parts) < 5 {
|
||||||
return newEntry
|
return newEntry
|
||||||
}
|
}
|
||||||
|
|
||||||
info := parts[4:]
|
newEntry.Fields = fields
|
||||||
|
|
||||||
if newEntry.Fields == nil {
|
|
||||||
newEntry.Fields = make(map[string]any)
|
|
||||||
}
|
|
||||||
newEntry.Fields["log_level"] = parts[0]
|
|
||||||
newEntry.Fields["message_date"] = parts[1]
|
|
||||||
newEntry.Fields["message_time"] = parts[2]
|
|
||||||
newEntry.Fields["transfer_id"] = parts[3]
|
|
||||||
newEntry.Fields["log_message"] = strings.Join(info, " ")
|
|
||||||
|
|
||||||
switch info[0] {
|
|
||||||
case "in:":
|
|
||||||
newEntry.Fields["log_type"] = "direction_info"
|
|
||||||
newEntry.Fields["transfer_direction"] = "incoming"
|
|
||||||
newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
|
||||||
case "out:":
|
|
||||||
newEntry.Fields["log_type"] = "direction_info"
|
|
||||||
newEntry.Fields["transfer_direction"] = "outgoing"
|
|
||||||
newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
|
||||||
case "queue-stats:":
|
|
||||||
newEntry.Fields["log_type"] = "queue-stats"
|
|
||||||
newEntry.Fields["queue-stats"] = strings.Join(info[1:], " ")
|
|
||||||
case "transfer:":
|
|
||||||
newEntry.Fields["log_type"] = "transfer_info"
|
|
||||||
newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
|
||||||
default:
|
|
||||||
newEntry.Fields["log_type"] = "log_message"
|
|
||||||
newEntry.Fields["transfer_info"] = strings.Join(info, " ")
|
|
||||||
}
|
|
||||||
|
|
||||||
return newEntry
|
return newEntry
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// func parseTixstreamService(entry LogEntry) LogEntry {
|
||||||
|
// newEntry := entry
|
||||||
|
// msg := strings.ReplaceAll(entry.Message, " ", " ")
|
||||||
|
// parts := strings.Fields(msg)
|
||||||
|
// if len(parts) < 5 {
|
||||||
|
// return newEntry
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// info := parts[4:]
|
||||||
|
//
|
||||||
|
// if newEntry.Fields == nil {
|
||||||
|
// newEntry.Fields = make(map[string]any)
|
||||||
|
// }
|
||||||
|
// newEntry.Fields["log_level"] = parts[0]
|
||||||
|
// newEntry.Fields["message_date"] = parts[1]
|
||||||
|
// newEntry.Fields["message_time"] = parts[2]
|
||||||
|
// newEntry.Fields["transfer_id"] = parts[3]
|
||||||
|
// newEntry.Fields["log_message"] = strings.Join(info, " ")
|
||||||
|
//
|
||||||
|
// switch info[0] {
|
||||||
|
// case "in:":
|
||||||
|
// newEntry.Fields["log_type"] = "log_message"
|
||||||
|
// newEntry.Fields["transfer_direction"] = "incoming"
|
||||||
|
// newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||||
|
// case "out:":
|
||||||
|
// newEntry.Fields["log_type"] = "log_message"
|
||||||
|
// newEntry.Fields["transfer_direction"] = "outgoing"
|
||||||
|
// newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||||
|
// case "queue-stats:":
|
||||||
|
// newEntry.Fields["log_type"] = "queue-stats"
|
||||||
|
// newEntry.Fields["queue-stats"] = strings.Join(info[1:], " ")
|
||||||
|
// case "transfer:":
|
||||||
|
// newEntry.Fields["log_type"] = "transfer_info"
|
||||||
|
// newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||||
|
// default:
|
||||||
|
// newEntry.Fields["log_type"] = "log_message"
|
||||||
|
// newEntry.Fields["transfer_info"] = strings.Join(info, " ")
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// return newEntry
|
||||||
|
// }
|
||||||
|
|
||||||
func parseTJMService(entry LogEntry) LogEntry {
|
func parseTJMService(entry LogEntry) LogEntry {
|
||||||
newEntry := entry
|
newEntry := entry
|
||||||
msg := strings.TrimSpace(entry.Message)
|
logContent := entry.Message
|
||||||
|
msg := strings.TrimSpace(logContent)
|
||||||
msg = strings.ReplaceAll(msg, " ", " ")
|
msg = strings.ReplaceAll(msg, " ", " ")
|
||||||
msg = strings.ReplaceAll(msg, "---", "")
|
msg = strings.ReplaceAll(msg, "---", "")
|
||||||
msg = strings.ReplaceAll(msg, " ", " ")
|
msg = strings.ReplaceAll(msg, " ", " ")
|
||||||
|
|
@ -269,43 +319,135 @@ func parseTJMService(entry LogEntry) LogEntry {
|
||||||
if len(parts) < 4 {
|
if len(parts) < 4 {
|
||||||
return newEntry
|
return newEntry
|
||||||
}
|
}
|
||||||
|
fields := make(map[string]any)
|
||||||
info := parts[3:]
|
matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||||
|
if len(matches) > 0 {
|
||||||
if newEntry.Fields == nil {
|
timestamp := strings.Join(strings.Split(matches[1], " "), "T")
|
||||||
newEntry.Fields = make(map[string]any)
|
fields["message_timestamp"] = timestamp
|
||||||
|
fields["log_level"] = strings.TrimSpace(matches[2])
|
||||||
|
fields["pid"] = strings.TrimSpace(matches[3])
|
||||||
|
fields["correlation_id"] = strings.TrimSpace(matches[4])
|
||||||
|
fields["username"] = strings.TrimSpace(matches[5])
|
||||||
|
fields["thread_id"] = strings.TrimSpace(matches[6])
|
||||||
|
fields["java_class"] = strings.TrimSpace(matches[7])
|
||||||
|
fields["log_message"] = strings.TrimSpace(matches[8])
|
||||||
|
} else {
|
||||||
|
fields["log_message"] = logContent
|
||||||
}
|
}
|
||||||
newEntry.Fields["log_level"] = parts[2]
|
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(fields["log_message"].(string))
|
||||||
newEntry.Fields["message_date"] = parts[0]
|
if len(trNameMatch) > 0 {
|
||||||
newEntry.Fields["message_time"] = parts[1]
|
fields["transfer_name"] = trNameMatch[1]
|
||||||
newEntry.Fields["message"] = strings.Join(info, " ")
|
fields["log_message"] = trNameMatch[2]
|
||||||
|
if strings.Contains(trNameMatch[1], "-in") {
|
||||||
tmpInfo := strings.ReplaceAll(strings.Join(info, " "), "[ ]", "[]")
|
fields["transfer_direction"] = "incoming"
|
||||||
tmpInfo = strings.ReplaceAll(tmpInfo, "[ ", "[")
|
}
|
||||||
tmpSplit := strings.Fields(tmpInfo)
|
if strings.Contains(trNameMatch[1], "-out") {
|
||||||
if len(tmpSplit) > 4 {
|
fields["transfer_direction"] = "outgoing"
|
||||||
newEntry.Fields["username"] = tmpSplit[2]
|
|
||||||
newEntry.Fields["correlation_id"] = tmpSplit[1]
|
|
||||||
newEntry.Fields["thread_id"] = tmpSplit[3]
|
|
||||||
newEntry.Fields["java_class"] = tmpSplit[4]
|
|
||||||
if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-out") {
|
|
||||||
newEntry.Fields["transfer_direction"] = "outgoing"
|
|
||||||
newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
|
||||||
} else if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-in") {
|
|
||||||
newEntry.Fields["transfer_direction"] = "incoming"
|
|
||||||
newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
|
||||||
} else {
|
|
||||||
newEntry.Fields["log_message"] = strings.Join(tmpSplit[6:], " ")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
value, ok := newEntry.Fields["log_message"]
|
||||||
|
if ok {
|
||||||
|
matches := tjmInnerLogPattern.FindStringSubmatch(value.(string))
|
||||||
|
groups := tjmInnerLogPattern.SubexpNames()
|
||||||
|
if len(matches) >= 1 {
|
||||||
|
for i, name := range groups {
|
||||||
|
if i != 0 && name != "" {
|
||||||
|
newEntry.Fields[name] = matches[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if strings.Contains(value.(string), "TransferJobTixstreamFile") || strings.Contains(value.(string), "PeerJobController.handlePeerJobAction") {
|
||||||
|
if strings.Contains(value.(string), "started transfer session") {
|
||||||
|
tmpSplit := strings.Split(value.(string), " ")
|
||||||
|
newEntry.Fields["transfer_name"] = tmpSplit[0]
|
||||||
|
newEntry.Fields["java_class_method"] = tmpSplit[2]
|
||||||
|
transferID, _ := strings.CutPrefix(strings.Join(tmpSplit[3:], " "), "started transfer session")
|
||||||
|
newEntry.Fields["transfer-id"] = strings.Split(transferID, " ")[0]
|
||||||
|
} else if strings.Contains(value.(string), "set transfer session id") {
|
||||||
|
tmpSplit := strings.Split(value.(string), " ")
|
||||||
|
newEntry.Fields["java_class_method"] = tmpSplit[0]
|
||||||
|
transferID := tmpSplit[len(tmpSplit)-1]
|
||||||
|
newEntry.Fields["transfer-id"] = transferID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
newEntry.Fields = fields
|
||||||
|
|
||||||
return newEntry
|
return newEntry
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
// func parseTJMService(entry LogEntry) LogEntry {
|
||||||
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
// newEntry := entry
|
||||||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
// msg := strings.TrimSpace(entry.Message)
|
||||||
)
|
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||||
|
// msg = strings.ReplaceAll(msg, "---", "")
|
||||||
|
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||||
|
// parts := strings.Fields(msg)
|
||||||
|
// if len(parts) < 4 {
|
||||||
|
// return newEntry
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// info := parts[3:]
|
||||||
|
//
|
||||||
|
// if newEntry.Fields == nil {
|
||||||
|
// newEntry.Fields = make(map[string]any)
|
||||||
|
// }
|
||||||
|
// newEntry.Fields["log_level"] = parts[2]
|
||||||
|
// newEntry.Fields["message_date"] = parts[0]
|
||||||
|
// newEntry.Fields["message_time"] = parts[1]
|
||||||
|
// newEntry.Fields["message"] = strings.Join(info, " ")
|
||||||
|
//
|
||||||
|
// tmpInfo := strings.ReplaceAll(strings.Join(info, " "), "[ ]", "[]")
|
||||||
|
// tmpInfo = strings.ReplaceAll(tmpInfo, "[ ", "[")
|
||||||
|
// tmpSplit := strings.Fields(tmpInfo)
|
||||||
|
// if len(tmpSplit) > 4 {
|
||||||
|
// newEntry.Fields["username"] = tmpSplit[2]
|
||||||
|
// newEntry.Fields["correlation_id"] = tmpSplit[1]
|
||||||
|
// newEntry.Fields["thread_id"] = tmpSplit[3]
|
||||||
|
// newEntry.Fields["java_class"] = tmpSplit[4]
|
||||||
|
// newEntry.Fields["log_type"] = "log_message"
|
||||||
|
// if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-out") {
|
||||||
|
// newEntry.Fields["transfer_direction"] = "outgoing"
|
||||||
|
// newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
||||||
|
// } else if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-in") {
|
||||||
|
// newEntry.Fields["transfer_direction"] = "incoming"
|
||||||
|
// newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
||||||
|
// } else {
|
||||||
|
// newEntry.Fields["log_message"] = strings.Join(tmpSplit[6:], " ")
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// value, ok := newEntry.Fields["log_message"]
|
||||||
|
// if ok {
|
||||||
|
// matches := tjmInnerLogPattern.FindStringSubmatch(value.(string))
|
||||||
|
// groups := tjmInnerLogPattern.SubexpNames()
|
||||||
|
// if len(matches) >= 1 {
|
||||||
|
// for i, name := range groups {
|
||||||
|
// if i != 0 && name != "" {
|
||||||
|
// newEntry.Fields[name] = matches[i]
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// } else {
|
||||||
|
// if strings.Contains(value.(string), "TransferJobTixstreamFile") || strings.Contains(value.(string), "PeerJobController.handlePeerJobAction") {
|
||||||
|
// if strings.Contains(value.(string), "started transfer session") {
|
||||||
|
// tmpSplit := strings.Split(value.(string), " ")
|
||||||
|
// newEntry.Fields["transfer_name"] = tmpSplit[0]
|
||||||
|
// newEntry.Fields["java_class_method"] = tmpSplit[2]
|
||||||
|
// transferID, _ := strings.CutPrefix(strings.Join(tmpSplit[3:], " "), "started transfer session")
|
||||||
|
// newEntry.Fields["transfer-id"] = strings.Split(transferID, " ")[0]
|
||||||
|
// } else if strings.Contains(value.(string), "set transfer session id") {
|
||||||
|
// tmpSplit := strings.Split(value.(string), " ")
|
||||||
|
// newEntry.Fields["java_class_method"] = tmpSplit[0]
|
||||||
|
// transferID := tmpSplit[len(tmpSplit)-1]
|
||||||
|
// newEntry.Fields["transfer-id"] = transferID
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// return newEntry
|
||||||
|
// }
|
||||||
|
|
||||||
func parseAMService(entry LogEntry) LogEntry {
|
func parseAMService(entry LogEntry) LogEntry {
|
||||||
newEntry := entry
|
newEntry := entry
|
||||||
|
|
@ -318,6 +460,26 @@ func parseAMService(entry LogEntry) LogEntry {
|
||||||
return newEntry
|
return newEntry
|
||||||
}
|
}
|
||||||
|
|
||||||
|
newEntry.Fields["log_level"] = matches[2]
|
||||||
|
newEntry.Fields["process_id"] = matches[3]
|
||||||
|
newEntry.Fields["thread_name"] = strings.TrimSpace(matches[4])
|
||||||
|
newEntry.Fields["logger_name"] = matches[5]
|
||||||
|
newEntry.Fields["log_message"] = matches[6]
|
||||||
|
|
||||||
|
return newEntry
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseTCCService(entry LogEntry) LogEntry {
|
||||||
|
newEntry := entry
|
||||||
|
if newEntry.Fields == nil {
|
||||||
|
newEntry.Fields = make(map[string]any)
|
||||||
|
}
|
||||||
|
|
||||||
|
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(entry.Message))
|
||||||
|
if len(matches) != 7 {
|
||||||
|
return newEntry
|
||||||
|
}
|
||||||
|
|
||||||
newEntry.Fields["timestamp"] = matches[1]
|
newEntry.Fields["timestamp"] = matches[1]
|
||||||
newEntry.Fields["log_level"] = matches[2]
|
newEntry.Fields["log_level"] = matches[2]
|
||||||
newEntry.Fields["process_id"] = matches[3]
|
newEntry.Fields["process_id"] = matches[3]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue