feat(service-monitor): update search pattern for better results
This commit is contained in:
parent
81d32a638f
commit
a8eb0ad726
1 changed files with 225 additions and 63 deletions
|
|
@ -212,56 +212,106 @@ func (jep *JournalEntryParser) parseServiceSpecific(entry LogEntry) LogEntry {
|
|||
return parseNginxService(entry)
|
||||
case "access-manager":
|
||||
return parseAMService(entry)
|
||||
case "tixel-control-center":
|
||||
return parseTCCService(entry)
|
||||
default:
|
||||
return entry
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||
tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||
tjmServicePattern = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
|
||||
tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-[A-Za-z]+-(?:in|out)) ?: (.*)$`)
|
||||
tsServicePattern = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
|
||||
tsTransferIDPattern = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
|
||||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
||||
tjmInnerLogPattern = regexp.MustCompile(`^(?P<transfername>[^ ]+) : (?P<javaclass>\w+)\.(?P<methode>\w+): started transfer session (?P<transferid>[a-f0-9\-]+) on (?P<localaddr>[\d\.]+:\d+) with target address (?P<targetaddr>[a-zA-Z0-9\.\-]+:\d+)`)
|
||||
)
|
||||
|
||||
func parseTixstreamService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
msg := strings.ReplaceAll(entry.Message, " ", " ")
|
||||
fields := make(map[string]any)
|
||||
matches := tsServicePattern.FindStringSubmatch(newEntry.Message)
|
||||
if len(matches) > 0 {
|
||||
timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
fields["log_level"] = strings.TrimSpace(matches[1])
|
||||
fields["message_timestamp"] = timestamp
|
||||
fields["log_message"] = strings.TrimSpace(matches[3])
|
||||
} else {
|
||||
fields["log_message"] = newEntry.Message
|
||||
}
|
||||
trNameMatch := tsTransferIDPattern.FindStringSubmatch(fields["log_message"].(string))
|
||||
if len(trNameMatch) > 0 {
|
||||
fields["transfer_id"] = trNameMatch[1]
|
||||
fields["log_message"] = trNameMatch[2]
|
||||
split := strings.Fields(trNameMatch[2])
|
||||
switch split[0] {
|
||||
case "in:":
|
||||
fields["transfer_direction"] = "incoming"
|
||||
case "out:":
|
||||
fields["transfer_direction"] = "outgoing"
|
||||
}
|
||||
}
|
||||
|
||||
msg := strings.ReplaceAll(newEntry.Message, " ", " ")
|
||||
parts := strings.Fields(msg)
|
||||
|
||||
if len(parts) < 5 {
|
||||
return newEntry
|
||||
}
|
||||
|
||||
info := parts[4:]
|
||||
|
||||
if newEntry.Fields == nil {
|
||||
newEntry.Fields = make(map[string]any)
|
||||
}
|
||||
newEntry.Fields["log_level"] = parts[0]
|
||||
newEntry.Fields["message_date"] = parts[1]
|
||||
newEntry.Fields["message_time"] = parts[2]
|
||||
newEntry.Fields["transfer_id"] = parts[3]
|
||||
newEntry.Fields["log_message"] = strings.Join(info, " ")
|
||||
|
||||
switch info[0] {
|
||||
case "in:":
|
||||
newEntry.Fields["log_type"] = "direction_info"
|
||||
newEntry.Fields["transfer_direction"] = "incoming"
|
||||
newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||
case "out:":
|
||||
newEntry.Fields["log_type"] = "direction_info"
|
||||
newEntry.Fields["transfer_direction"] = "outgoing"
|
||||
newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||
case "queue-stats:":
|
||||
newEntry.Fields["log_type"] = "queue-stats"
|
||||
newEntry.Fields["queue-stats"] = strings.Join(info[1:], " ")
|
||||
case "transfer:":
|
||||
newEntry.Fields["log_type"] = "transfer_info"
|
||||
newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||
default:
|
||||
newEntry.Fields["log_type"] = "log_message"
|
||||
newEntry.Fields["transfer_info"] = strings.Join(info, " ")
|
||||
}
|
||||
|
||||
newEntry.Fields = fields
|
||||
return newEntry
|
||||
}
|
||||
|
||||
// func parseTixstreamService(entry LogEntry) LogEntry {
|
||||
// newEntry := entry
|
||||
// msg := strings.ReplaceAll(entry.Message, " ", " ")
|
||||
// parts := strings.Fields(msg)
|
||||
// if len(parts) < 5 {
|
||||
// return newEntry
|
||||
// }
|
||||
//
|
||||
// info := parts[4:]
|
||||
//
|
||||
// if newEntry.Fields == nil {
|
||||
// newEntry.Fields = make(map[string]any)
|
||||
// }
|
||||
// newEntry.Fields["log_level"] = parts[0]
|
||||
// newEntry.Fields["message_date"] = parts[1]
|
||||
// newEntry.Fields["message_time"] = parts[2]
|
||||
// newEntry.Fields["transfer_id"] = parts[3]
|
||||
// newEntry.Fields["log_message"] = strings.Join(info, " ")
|
||||
//
|
||||
// switch info[0] {
|
||||
// case "in:":
|
||||
// newEntry.Fields["log_type"] = "log_message"
|
||||
// newEntry.Fields["transfer_direction"] = "incoming"
|
||||
// newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||
// case "out:":
|
||||
// newEntry.Fields["log_type"] = "log_message"
|
||||
// newEntry.Fields["transfer_direction"] = "outgoing"
|
||||
// newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||
// case "queue-stats:":
|
||||
// newEntry.Fields["log_type"] = "queue-stats"
|
||||
// newEntry.Fields["queue-stats"] = strings.Join(info[1:], " ")
|
||||
// case "transfer:":
|
||||
// newEntry.Fields["log_type"] = "transfer_info"
|
||||
// newEntry.Fields["transfer_info"] = strings.Join(info[1:], " ")
|
||||
// default:
|
||||
// newEntry.Fields["log_type"] = "log_message"
|
||||
// newEntry.Fields["transfer_info"] = strings.Join(info, " ")
|
||||
// }
|
||||
//
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseTJMService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
msg := strings.TrimSpace(entry.Message)
|
||||
logContent := entry.Message
|
||||
msg := strings.TrimSpace(logContent)
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
msg = strings.ReplaceAll(msg, "---", "")
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
|
|
@ -269,43 +319,135 @@ func parseTJMService(entry LogEntry) LogEntry {
|
|||
if len(parts) < 4 {
|
||||
return newEntry
|
||||
}
|
||||
|
||||
info := parts[3:]
|
||||
|
||||
if newEntry.Fields == nil {
|
||||
newEntry.Fields = make(map[string]any)
|
||||
fields := make(map[string]any)
|
||||
matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||
if len(matches) > 0 {
|
||||
timestamp := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
fields["message_timestamp"] = timestamp
|
||||
fields["log_level"] = strings.TrimSpace(matches[2])
|
||||
fields["pid"] = strings.TrimSpace(matches[3])
|
||||
fields["correlation_id"] = strings.TrimSpace(matches[4])
|
||||
fields["username"] = strings.TrimSpace(matches[5])
|
||||
fields["thread_id"] = strings.TrimSpace(matches[6])
|
||||
fields["java_class"] = strings.TrimSpace(matches[7])
|
||||
fields["log_message"] = strings.TrimSpace(matches[8])
|
||||
} else {
|
||||
fields["log_message"] = logContent
|
||||
}
|
||||
newEntry.Fields["log_level"] = parts[2]
|
||||
newEntry.Fields["message_date"] = parts[0]
|
||||
newEntry.Fields["message_time"] = parts[1]
|
||||
newEntry.Fields["message"] = strings.Join(info, " ")
|
||||
|
||||
tmpInfo := strings.ReplaceAll(strings.Join(info, " "), "[ ]", "[]")
|
||||
tmpInfo = strings.ReplaceAll(tmpInfo, "[ ", "[")
|
||||
tmpSplit := strings.Fields(tmpInfo)
|
||||
if len(tmpSplit) > 4 {
|
||||
newEntry.Fields["username"] = tmpSplit[2]
|
||||
newEntry.Fields["correlation_id"] = tmpSplit[1]
|
||||
newEntry.Fields["thread_id"] = tmpSplit[3]
|
||||
newEntry.Fields["java_class"] = tmpSplit[4]
|
||||
if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-out") {
|
||||
newEntry.Fields["transfer_direction"] = "outgoing"
|
||||
newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
||||
} else if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-in") {
|
||||
newEntry.Fields["transfer_direction"] = "incoming"
|
||||
newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
||||
} else {
|
||||
newEntry.Fields["log_message"] = strings.Join(tmpSplit[6:], " ")
|
||||
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(fields["log_message"].(string))
|
||||
if len(trNameMatch) > 0 {
|
||||
fields["transfer_name"] = trNameMatch[1]
|
||||
fields["log_message"] = trNameMatch[2]
|
||||
if strings.Contains(trNameMatch[1], "-in") {
|
||||
fields["transfer_direction"] = "incoming"
|
||||
}
|
||||
if strings.Contains(trNameMatch[1], "-out") {
|
||||
fields["transfer_direction"] = "outgoing"
|
||||
}
|
||||
}
|
||||
value, ok := newEntry.Fields["log_message"]
|
||||
if ok {
|
||||
matches := tjmInnerLogPattern.FindStringSubmatch(value.(string))
|
||||
groups := tjmInnerLogPattern.SubexpNames()
|
||||
if len(matches) >= 1 {
|
||||
for i, name := range groups {
|
||||
if i != 0 && name != "" {
|
||||
newEntry.Fields[name] = matches[i]
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if strings.Contains(value.(string), "TransferJobTixstreamFile") || strings.Contains(value.(string), "PeerJobController.handlePeerJobAction") {
|
||||
if strings.Contains(value.(string), "started transfer session") {
|
||||
tmpSplit := strings.Split(value.(string), " ")
|
||||
newEntry.Fields["transfer_name"] = tmpSplit[0]
|
||||
newEntry.Fields["java_class_method"] = tmpSplit[2]
|
||||
transferID, _ := strings.CutPrefix(strings.Join(tmpSplit[3:], " "), "started transfer session")
|
||||
newEntry.Fields["transfer-id"] = strings.Split(transferID, " ")[0]
|
||||
} else if strings.Contains(value.(string), "set transfer session id") {
|
||||
tmpSplit := strings.Split(value.(string), " ")
|
||||
newEntry.Fields["java_class_method"] = tmpSplit[0]
|
||||
transferID := tmpSplit[len(tmpSplit)-1]
|
||||
newEntry.Fields["transfer-id"] = transferID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newEntry.Fields = fields
|
||||
|
||||
return newEntry
|
||||
}
|
||||
|
||||
var (
|
||||
amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
|
||||
nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
|
||||
)
|
||||
// func parseTJMService(entry LogEntry) LogEntry {
|
||||
// newEntry := entry
|
||||
// msg := strings.TrimSpace(entry.Message)
|
||||
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||
// msg = strings.ReplaceAll(msg, "---", "")
|
||||
// msg = strings.ReplaceAll(msg, " ", " ")
|
||||
// parts := strings.Fields(msg)
|
||||
// if len(parts) < 4 {
|
||||
// return newEntry
|
||||
// }
|
||||
//
|
||||
// info := parts[3:]
|
||||
//
|
||||
// if newEntry.Fields == nil {
|
||||
// newEntry.Fields = make(map[string]any)
|
||||
// }
|
||||
// newEntry.Fields["log_level"] = parts[2]
|
||||
// newEntry.Fields["message_date"] = parts[0]
|
||||
// newEntry.Fields["message_time"] = parts[1]
|
||||
// newEntry.Fields["message"] = strings.Join(info, " ")
|
||||
//
|
||||
// tmpInfo := strings.ReplaceAll(strings.Join(info, " "), "[ ]", "[]")
|
||||
// tmpInfo = strings.ReplaceAll(tmpInfo, "[ ", "[")
|
||||
// tmpSplit := strings.Fields(tmpInfo)
|
||||
// if len(tmpSplit) > 4 {
|
||||
// newEntry.Fields["username"] = tmpSplit[2]
|
||||
// newEntry.Fields["correlation_id"] = tmpSplit[1]
|
||||
// newEntry.Fields["thread_id"] = tmpSplit[3]
|
||||
// newEntry.Fields["java_class"] = tmpSplit[4]
|
||||
// newEntry.Fields["log_type"] = "log_message"
|
||||
// if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-out") {
|
||||
// newEntry.Fields["transfer_direction"] = "outgoing"
|
||||
// newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
||||
// } else if len(tmpSplit) > 6 && strings.Contains(tmpSplit[6], "-in") {
|
||||
// newEntry.Fields["transfer_direction"] = "incoming"
|
||||
// newEntry.Fields["log_message"] = strings.Join(tmpSplit[7:], " ")
|
||||
// } else {
|
||||
// newEntry.Fields["log_message"] = strings.Join(tmpSplit[6:], " ")
|
||||
// }
|
||||
// }
|
||||
// value, ok := newEntry.Fields["log_message"]
|
||||
// if ok {
|
||||
// matches := tjmInnerLogPattern.FindStringSubmatch(value.(string))
|
||||
// groups := tjmInnerLogPattern.SubexpNames()
|
||||
// if len(matches) >= 1 {
|
||||
// for i, name := range groups {
|
||||
// if i != 0 && name != "" {
|
||||
// newEntry.Fields[name] = matches[i]
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// if strings.Contains(value.(string), "TransferJobTixstreamFile") || strings.Contains(value.(string), "PeerJobController.handlePeerJobAction") {
|
||||
// if strings.Contains(value.(string), "started transfer session") {
|
||||
// tmpSplit := strings.Split(value.(string), " ")
|
||||
// newEntry.Fields["transfer_name"] = tmpSplit[0]
|
||||
// newEntry.Fields["java_class_method"] = tmpSplit[2]
|
||||
// transferID, _ := strings.CutPrefix(strings.Join(tmpSplit[3:], " "), "started transfer session")
|
||||
// newEntry.Fields["transfer-id"] = strings.Split(transferID, " ")[0]
|
||||
// } else if strings.Contains(value.(string), "set transfer session id") {
|
||||
// tmpSplit := strings.Split(value.(string), " ")
|
||||
// newEntry.Fields["java_class_method"] = tmpSplit[0]
|
||||
// transferID := tmpSplit[len(tmpSplit)-1]
|
||||
// newEntry.Fields["transfer-id"] = transferID
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// return newEntry
|
||||
// }
|
||||
|
||||
func parseAMService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
|
|
@ -318,6 +460,26 @@ func parseAMService(entry LogEntry) LogEntry {
|
|||
return newEntry
|
||||
}
|
||||
|
||||
newEntry.Fields["log_level"] = matches[2]
|
||||
newEntry.Fields["process_id"] = matches[3]
|
||||
newEntry.Fields["thread_name"] = strings.TrimSpace(matches[4])
|
||||
newEntry.Fields["logger_name"] = matches[5]
|
||||
newEntry.Fields["log_message"] = matches[6]
|
||||
|
||||
return newEntry
|
||||
}
|
||||
|
||||
func parseTCCService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
if newEntry.Fields == nil {
|
||||
newEntry.Fields = make(map[string]any)
|
||||
}
|
||||
|
||||
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(entry.Message))
|
||||
if len(matches) != 7 {
|
||||
return newEntry
|
||||
}
|
||||
|
||||
newEntry.Fields["timestamp"] = matches[1]
|
||||
newEntry.Fields["log_level"] = matches[2]
|
||||
newEntry.Fields["process_id"] = matches[3]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue