refactor(tixel-watch): refactor watch-tool to enable type safety and better performance
This commit is contained in:
Patryk Hegenberg
parent
159df116c8
commit
25dffecb43
8 changed files with 367 additions and 201 deletions
|
|
@ -136,17 +136,18 @@ func (jep *JournalEntryParser) Parse(jsonLine string) (LogEntry, error) {
|
|||
}
|
||||
|
||||
if msg, ok := journalData["MESSAGE"].(string); ok {
|
||||
entry.Message = msg
|
||||
entry.LogMessage = msg
|
||||
}
|
||||
|
||||
if priority, ok := journalData["PRIORITY"].(string); ok {
|
||||
entry.Priority = priority
|
||||
entry.Fields["priority_name"] = jep.getPriorityName(priority)
|
||||
entry.PriorityName = jep.getPriorityName(priority)
|
||||
}
|
||||
|
||||
if pidStr, ok := journalData["_PID"].(string); ok {
|
||||
if pid, err := strconv.Atoi(pidStr); err == nil {
|
||||
entry.PID = pid
|
||||
entry.SyslogInfo.ProcessInfo = strconv.FormatInt(int64(pid), 10)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -197,7 +198,7 @@ func (jep *JournalEntryParser) extractSystemdFields(journalData map[string]any,
|
|||
for _, field := range systemdFields {
|
||||
if value, ok := journalData[field]; ok {
|
||||
esFieldName := strings.ToLower(strings.TrimPrefix(field, "_"))
|
||||
entry.Fields[esFieldName] = value
|
||||
entry.SyslogInfo.Fields[esFieldName] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -237,115 +238,114 @@ var (
|
|||
|
||||
func parseTixstreamService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
fields := make(map[string]any)
|
||||
var baseInfo TSBaseInfo
|
||||
|
||||
matches := tsServicePattern.FindStringSubmatch(newEntry.Message)
|
||||
matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(matches) > 0 {
|
||||
timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
fields["log_level"] = strings.TrimSpace(matches[1])
|
||||
fields["message_timestamp"] = timestamp
|
||||
fields["log_message"] = strings.TrimSpace(matches[3])
|
||||
} else {
|
||||
fields["log_message"] = newEntry.Message
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
newEntry.LogMessage = strings.TrimSpace(matches[3])
|
||||
}
|
||||
trNameMatch := tsTransferIDPattern.FindStringSubmatch(fields["log_message"].(string))
|
||||
trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
var transferID string
|
||||
if len(trNameMatch) > 0 {
|
||||
transferID = trNameMatch[1]
|
||||
fields["log_message"] = trNameMatch[2]
|
||||
newEntry.LogMessage = trNameMatch[2]
|
||||
split := strings.Fields(trNameMatch[2])
|
||||
switch split[0] {
|
||||
case "in:":
|
||||
fields["transfer_direction"] = "incoming"
|
||||
baseInfo.Direction = "incoming"
|
||||
case "out:":
|
||||
fields["transfer_direction"] = "outgoing"
|
||||
baseInfo.Direction = "outgoing"
|
||||
}
|
||||
}
|
||||
|
||||
msg := strings.ReplaceAll(newEntry.Message, " ", " ")
|
||||
msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ")
|
||||
parts := strings.Fields(msg)
|
||||
|
||||
if len(parts) < 5 {
|
||||
return newEntry
|
||||
}
|
||||
tsDetail := tsDetailPattern1.FindStringSubmatch(fields["log_message"].(string))
|
||||
tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
fields["thread"] = threadInt
|
||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
fields["buffers"] = buffersInt
|
||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
fields["file_count"] = fileCountInt
|
||||
fileSizeInt, _ := strconv.Atoi(tsDetail[4])
|
||||
fields["file_size"] = fileSizeInt
|
||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
fields["chunksize"] = chunkSizeInt
|
||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
fields["streams"] = streamsInt
|
||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
fields["target_datarate"] = datarateFloat
|
||||
fields["protocoll"] = tsDetail[8]
|
||||
fields["destination"] = tsDetail[9]
|
||||
fields["sender_id"] = tsDetail[10]
|
||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Buffers = buffersInt
|
||||
baseInfo.FileCount = fileCountInt
|
||||
baseInfo.FileSize = fileSizeFloat
|
||||
baseInfo.ChunkSize = chunkSizeInt
|
||||
baseInfo.Streams = streamsInt
|
||||
baseInfo.TargetDatarate = datarateFloat
|
||||
baseInfo.Protocoll = tsDetail[8]
|
||||
baseInfo.Destination = tsDetail[9]
|
||||
baseInfo.Sender = tsDetail[10]
|
||||
}
|
||||
tsDetail = tsDetailPattern2.FindStringSubmatch(fields["log_message"].(string))
|
||||
tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
fields["transfer_target"] = tsDetail[1]
|
||||
baseInfo.Target = tsDetail[1]
|
||||
}
|
||||
tsDetail = tsDetailPattern3.FindStringSubmatch(fields["log_message"].(string))
|
||||
tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
fields["thread"] = threadInt
|
||||
buffersInt, _ := strconv.Atoi(tsDetail[2])
|
||||
fields["buffers"] = buffersInt
|
||||
fileCountInt, _ := strconv.Atoi(tsDetail[3])
|
||||
fields["file_count"] = fileCountInt
|
||||
fileSizeInt, _ := strconv.Atoi(tsDetail[4])
|
||||
fields["file_size"] = fileSizeInt
|
||||
fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
|
||||
chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
|
||||
fields["chunksize"] = chunkSizeInt
|
||||
streamsInt, _ := strconv.Atoi(tsDetail[6])
|
||||
fields["streams"] = streamsInt
|
||||
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
|
||||
fields["target_datarate"] = datarateFloat
|
||||
fields["protocoll"] = tsDetail[8]
|
||||
fields["source"] = tsDetail[9]
|
||||
fields["receiver_id"] = tsDetail[10]
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Buffers = buffersInt
|
||||
baseInfo.FileCount = fileCountInt
|
||||
baseInfo.FileSize = fileSizeFloat
|
||||
baseInfo.ChunkSize = chunkSizeInt
|
||||
baseInfo.Streams = streamsInt
|
||||
baseInfo.TargetDatarate = datarateFloat
|
||||
baseInfo.Protocoll = tsDetail[8]
|
||||
baseInfo.Source = tsDetail[9]
|
||||
baseInfo.Receiver = tsDetail[10]
|
||||
}
|
||||
tsDetail = tsDetailPattern4.FindStringSubmatch(fields["log_message"].(string))
|
||||
tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(tsDetail) > 0 {
|
||||
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
|
||||
fields["thread"] = threadInt
|
||||
fields["source"] = tsDetail[2]
|
||||
fields["destination"] = tsDetail[3]
|
||||
fields["item"] = tsDetail[4]
|
||||
fields["count"] = tsDetail[5]
|
||||
baseInfo.Lane = threadInt
|
||||
baseInfo.Source = tsDetail[2]
|
||||
baseInfo.Destination = tsDetail[3]
|
||||
}
|
||||
if strings.Contains(fields["log_message"].(string), "Transfer start") || strings.Contains(fields["log_message"].(string), "Transfer started,") {
|
||||
fields["event"] = "transfer_started"
|
||||
fields["start_time"] = fields["message_timestamp"]
|
||||
if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
|
||||
baseInfo.StartTime = newEntry.Timestamp
|
||||
}
|
||||
if strings.Contains(fields["log_message"].(string), "Transfer stopped local state=finished") {
|
||||
fields["event"] = "transfer_stopped"
|
||||
fields["end_time"] = fields["message_timestamp"]
|
||||
if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
|
||||
baseInfo.EndTime = newEntry.Timestamp
|
||||
}
|
||||
// value, ok := fields["transfer_id"]
|
||||
if transferID != "" {
|
||||
fields["transfer_identifier"] = transferID
|
||||
baseInfo.TransferID = transferID
|
||||
} else {
|
||||
baseInfo.TransferID = "no_transfer_id"
|
||||
}
|
||||
if fields["transfer_identifier"] == nil {
|
||||
fields["transfer_identifier"] = "unknown"
|
||||
if !baseInfo.StartTime.IsZero() {
|
||||
newEntry.BaseInformation = baseInfo
|
||||
}
|
||||
if fields["message_timestamp"] == nil {
|
||||
fields["message_timestamp"] = newEntry.Timestamp
|
||||
}
|
||||
newEntry.Fields = fields
|
||||
return newEntry
|
||||
}
|
||||
|
||||
func parseTJMService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
logContent := entry.Message
|
||||
var baseInfo TJMBaseInfo
|
||||
|
||||
logContent := entry.LogMessage
|
||||
msg := strings.TrimSpace(logContent)
|
||||
msg = strings.ReplaceAll(msg, " ", " ")
|
||||
msg = strings.ReplaceAll(msg, "---", "")
|
||||
|
|
@ -354,156 +354,160 @@ func parseTJMService(entry LogEntry) LogEntry {
|
|||
if len(parts) < 4 {
|
||||
return newEntry
|
||||
}
|
||||
fields := make(map[string]any)
|
||||
matches := tjmServicePattern.FindStringSubmatch(logContent)
|
||||
if len(matches) > 0 {
|
||||
timestamp := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
fields["message_timestamp"] = timestamp
|
||||
fields["log_level"] = strings.TrimSpace(matches[2])
|
||||
fields["pid"] = strings.TrimSpace(matches[3])
|
||||
fields["correlation_id"] = strings.TrimSpace(matches[4])
|
||||
fields["username"] = strings.TrimSpace(matches[5])
|
||||
fields["thread_id"] = strings.TrimSpace(matches[6])
|
||||
fields["java_class"] = strings.TrimSpace(matches[7])
|
||||
fields["log_message"] = strings.TrimSpace(matches[8])
|
||||
timestamp := strings.Join(strings.Split(matches[2], " "), "T")
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[1])
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
newEntry.LogLevel = strings.TrimSpace(matches[2])
|
||||
newEntry.LogMessage = strings.TrimSpace(matches[8])
|
||||
baseInfo = TJMBaseInfo{
|
||||
ProcessID: strings.TrimSpace(matches[3]),
|
||||
CorrelationID: strings.TrimSpace(matches[4]),
|
||||
Username: strings.TrimSpace(matches[5]),
|
||||
ThreadID: strings.TrimSpace(matches[6]),
|
||||
JavaClass: strings.TrimSpace(matches[7]),
|
||||
}
|
||||
} else {
|
||||
fields["log_message"] = logContent
|
||||
newEntry.LogMessage = logContent
|
||||
}
|
||||
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(fields["log_message"].(string))
|
||||
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
|
||||
var transferName string
|
||||
var transferID string
|
||||
if len(trNameMatch) > 0 {
|
||||
transferName = trNameMatch[1]
|
||||
fields["log_message"] = trNameMatch[2]
|
||||
newEntry.LogMessage = trNameMatch[2]
|
||||
if strings.Contains(trNameMatch[1], "-in") {
|
||||
fields["transfer_direction"] = "incoming"
|
||||
baseInfo.Direction = "incoming"
|
||||
}
|
||||
if strings.Contains(trNameMatch[1], "-out") {
|
||||
fields["transfer_direction"] = "outgoing"
|
||||
baseInfo.Direction = "outgoing"
|
||||
}
|
||||
}
|
||||
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(fields["log_message"].(string))
|
||||
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(trIDMatch) > 0 {
|
||||
transferID = trIDMatch[1]
|
||||
}
|
||||
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(fields["log_message"].(string))
|
||||
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
|
||||
if len(trIDMatch) > 0 {
|
||||
transferID = trIDMatch[2]
|
||||
}
|
||||
// value, ok := fields["transfer_id"]
|
||||
if transferID != "" {
|
||||
fields["transfer_identifier"] = transferID
|
||||
baseInfo.TransferID = transferID
|
||||
} else if transferName != "" {
|
||||
// value, ok := fields["transfer_name"]
|
||||
// if ok {
|
||||
fields["transfer_identifier"] = transferName
|
||||
// }
|
||||
baseInfo.TransferID = transferName
|
||||
} else {
|
||||
baseInfo.TransferID = "no_transfer_id"
|
||||
}
|
||||
if fields["transfer_identifier"] == nil {
|
||||
fields["transfer_identifier"] = "unknown"
|
||||
}
|
||||
if fields["message_timestamp"] == nil {
|
||||
fields["message_timestamp"] = newEntry.Timestamp
|
||||
}
|
||||
|
||||
newEntry.Fields = fields
|
||||
newEntry.BaseInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
|
||||
func parseAMService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
if newEntry.Fields == nil {
|
||||
newEntry.Fields = make(map[string]any)
|
||||
}
|
||||
fields := make(map[string]any)
|
||||
logContent := newEntry.LogMessage
|
||||
|
||||
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(entry.Message))
|
||||
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
|
||||
if len(matches) != 7 {
|
||||
return newEntry
|
||||
}
|
||||
timestamp := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
fields["message_timestamp"] = timestamp
|
||||
fields["log_level"] = matches[2]
|
||||
fields["process_id"] = matches[3]
|
||||
fields["thread_id"] = strings.TrimSpace(matches[4])
|
||||
fields["logger_name"] = matches[5]
|
||||
fields["log_message"] = matches[6]
|
||||
|
||||
fields["transfer_identifier"] = "unknown"
|
||||
if fields["message_timestamp"] == nil {
|
||||
fields["message_timestamp"] = newEntry.Timestamp
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
newEntry.Fields = fields
|
||||
baseInfo := AMBaseInfo{
|
||||
ProcessID: matches[3],
|
||||
ThreadID: strings.TrimSpace(matches[4]),
|
||||
LoggerName: matches[5],
|
||||
}
|
||||
newEntry.LogLevel = matches[2]
|
||||
newEntry.LogMessage = matches[6]
|
||||
newEntry.BaseInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
|
||||
func parseTCCService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
if newEntry.Fields == nil {
|
||||
newEntry.Fields = make(map[string]any)
|
||||
}
|
||||
logContent := newEntry.LogMessage
|
||||
|
||||
fields := make(map[string]any)
|
||||
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(entry.Message))
|
||||
matches := tccServicePattern.FindStringSubmatch(logContent)
|
||||
if len(matches) != 7 {
|
||||
return newEntry
|
||||
}
|
||||
timestamp := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
fields["message_timestamp"] = timestamp
|
||||
fields["log_level"] = matches[2]
|
||||
fields["process_id"] = matches[3]
|
||||
fields["thread_id"] = strings.TrimSpace(matches[4])
|
||||
fields["logger_name"] = matches[5]
|
||||
fields["log_message"] = matches[6]
|
||||
|
||||
fields["transfer_identifier"] = "unknown"
|
||||
if fields["message_timestamp"].(string) == "" {
|
||||
fields["message_timestamp"] = newEntry.Timestamp
|
||||
timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
|
||||
if newEntry.Timestamp.IsZero() {
|
||||
timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
|
||||
if err != nil {
|
||||
slog.Error("cant parse time string", "error", err)
|
||||
}
|
||||
newEntry.Timestamp = timeParsed
|
||||
}
|
||||
newEntry.Fields = fields
|
||||
|
||||
newEntry.Fields["timestamp"] = matches[1]
|
||||
newEntry.Fields["log_level"] = matches[2]
|
||||
newEntry.Fields["process_id"] = matches[3]
|
||||
newEntry.Fields["thread_name"] = strings.TrimSpace(matches[4])
|
||||
newEntry.Fields["logger_name"] = matches[5]
|
||||
newEntry.Fields["log_message"] = matches[6]
|
||||
baseInfo := TCCBaseInfo{
|
||||
ProcessID: matches[3],
|
||||
ThreadID: strings.TrimSpace(matches[4]),
|
||||
LoggerName: matches[5],
|
||||
}
|
||||
newEntry.LogLevel = matches[2]
|
||||
newEntry.LogMessage = matches[6]
|
||||
newEntry.BaseInformation = baseInfo
|
||||
|
||||
return newEntry
|
||||
}
|
||||
|
||||
func parseNginxService(entry LogEntry) LogEntry {
|
||||
newEntry := entry
|
||||
if newEntry.Fields == nil {
|
||||
newEntry.Fields = make(map[string]any)
|
||||
}
|
||||
|
||||
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.Message))
|
||||
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
|
||||
if len(matches) < 7 {
|
||||
return newEntry
|
||||
}
|
||||
|
||||
newEntry.Fields["client_ip"] = matches[1]
|
||||
newEntry.Fields["remote_user"] = matches[2]
|
||||
newEntry.Fields["timestamp"] = matches[3]
|
||||
newEntry.Fields["request"] = matches[4]
|
||||
newEntry.Fields["status_code"] = matches[5]
|
||||
newEntry.Fields["bytes_sent"] = matches[6]
|
||||
statusCode, err := strconv.ParseInt(matches[5], 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant parse statuscode", "error", err)
|
||||
}
|
||||
bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
|
||||
if err != nil {
|
||||
slog.Error("cant parse bytessend", "error", err)
|
||||
}
|
||||
baseInfo := NGinXBaseInfo{
|
||||
ClientIP: matches[1],
|
||||
RemoteUser: matches[2],
|
||||
Request: matches[4],
|
||||
StatusCode: int(statusCode),
|
||||
BytesSend: int(bytesSend),
|
||||
}
|
||||
|
||||
if len(matches) > 7 && matches[7] != "" {
|
||||
newEntry.Fields["referer"] = matches[7]
|
||||
baseInfo.Referer = matches[7]
|
||||
}
|
||||
if len(matches) > 8 && matches[8] != "" {
|
||||
newEntry.Fields["user_agent"] = matches[8]
|
||||
baseInfo.UserAgent = matches[8]
|
||||
}
|
||||
|
||||
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
|
||||
newEntry.Fields["http_method"] = requestParts[0]
|
||||
newEntry.Fields["request_uri"] = requestParts[1]
|
||||
newEntry.Fields["http_version"] = requestParts[2]
|
||||
baseInfo.HTTPMethod = requestParts[0]
|
||||
baseInfo.RequestURI = requestParts[1]
|
||||
baseInfo.HTTPVersion = requestParts[2]
|
||||
}
|
||||
|
||||
return newEntry
|
||||
}
|
||||
|
||||
func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
|
||||
if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
|
||||
timeStr += "Z"
|
||||
}
|
||||
return time.Parse(time.RFC3339Nano, timeStr)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue