git/watch-tool
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tixel-watch): refactor watch-tool to enable type safety and better performance

Browse source
This commit is contained in:
Patryk Hegenberg 2025-09-21 22:36:05 +02:00
parent 159df116c8
commit 25dffecb43
8 changed files with 367 additions and 201 deletions
Download patch file Download diff file Expand all files Collapse all files

7
config.go
View file

@ -36,6 +36,7 @@ type ServiceConfig struct {
} }
type ElasticsearchConfig struct { type ElasticsearchConfig struct {
Enabled bool `mapstructure:"enabled"`
URL string `mapstructure:"url"` URL string `mapstructure:"url"`
Index string `mapstructure:"index"` Index string `mapstructure:"index"`
Username string `mapstructure:"username"` Username string `mapstructure:"username"`
@ -43,6 +44,11 @@ type ElasticsearchConfig struct {
Timeout int `mapstructure:"timeout"` Timeout int `mapstructure:"timeout"`
} }
type LocalStorage struct {
Enable bool `mapstructure:"enabled"`
DBPath string `mapstructure:"db_path"`
}
type SystemMetrics struct { type SystemMetrics struct {
Enabled bool `mapstructure:"enabled"` Enabled bool `mapstructure:"enabled"`
CollectCPU bool `mapstructure:"collect_cpu"` CollectCPU bool `mapstructure:"collect_cpu"`
@ -66,6 +72,7 @@ type SystemMetrics struct {
type Config struct { type Config struct {
Elasticsearch ElasticsearchConfig `mapstructure:"elasticsearch"` Elasticsearch ElasticsearchConfig `mapstructure:"elasticsearch"`
LocalStorage LocalStorage `mapstrucutre:"localstorage"`
Tools []ToolConfig `mapstructure:"tools"` Tools []ToolConfig `mapstructure:"tools"`
Services []ServiceConfig `mapstructure:"services"` Services []ServiceConfig `mapstructure:"services"`
PollIntervalSeconds int `mapstructure:"poll_interval_seconds"` PollIntervalSeconds int `mapstructure:"poll_interval_seconds"`

10
elasticsearch.go
View file

@ -102,12 +102,18 @@ func (esc *ElasticsearchClient) SendBatch(baseIndex string, entries []LogEntry)
} }
func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics SystemResources) error { func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics SystemResources) error {
data, err := json.Marshal(metrics) msg := LogMessage{
Service: "system-metrics",
Timestamp: time.Now(),
LogLevel: "Info",
SystemMetrics: metrics,
}
data, err := json.Marshal(msg)
if err != nil { if err != nil {
return fmt.Errorf("JSON marshalling error: %w", err) return fmt.Errorf("JSON marshalling error: %w", err)
} }
systemIndex := fmt.Sprintf("%s-system", baseIndex) systemIndex := "tixel"
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel() defer cancel()

20
elasticsearch_exporter.go
View file

@ -31,7 +31,7 @@ type ExportResult struct {
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
} }
func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []string, batchSize int, writer io.Writer) error { func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []string, batchSize int, since int, writer io.Writer) error {
startTime := time.Now() startTime := time.Now()
if _, err := writer.Write([]byte("{\n \"export_info\": {\n")); err != nil { if _, err := writer.Write([]byte("{\n \"export_info\": {\n")); err != nil {
@ -42,6 +42,7 @@ func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []st
"timestamp": startTime, "timestamp": startTime,
"indices": indices, "indices": indices,
"batch_size": batchSize, "batch_size": batchSize,
"sinceDays": since,
} }
infoBytes, err := json.MarshalIndent(exportInfo, " ", " ") infoBytes, err := json.MarshalIndent(exportInfo, " ", " ")
@ -72,7 +73,7 @@ func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []st
} }
first = false first = false
result := e.exportIndex(ctx, index, batchSize, writer) result := e.exportIndex(ctx, index, batchSize, since, writer)
results = append(results, result) results = append(results, result)
if result.Error != "" { if result.Error != "" {
@ -98,7 +99,7 @@ func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []st
return nil return nil
} }
func (e *ElasticsearchExporter) exportIndex(ctx context.Context, index string, batchSize int, writer io.Writer) ExportResult { func (e *ElasticsearchExporter) exportIndex(ctx context.Context, index string, batchSize int, since int, writer io.Writer) ExportResult {
startTime := time.Now() startTime := time.Now()
result := ExportResult{ result := ExportResult{
Index: index, Index: index,
@ -113,7 +114,18 @@ func (e *ElasticsearchExporter) exportIndex(ctx context.Context, index string, b
} }
query := `{"query":{"match_all":{}}}` query := `{"query":{"match_all":{}}}`
if since > 0 {
query = fmt.Sprintf(`{
"query": {
"range": {
"timestamp": {
"gte": "now-%dd/d",
"lt": "now/d"
}
}
}
}`, since)
}
res, err := e.client.Search( res, err := e.client.Search(
e.client.Search.WithContext(ctx), e.client.Search.WithContext(ctx),
e.client.Search.WithIndex(index), e.client.Search.WithIndex(index),

42
file_monitor.go
View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"regexp" "regexp"
"strconv"
"strings" "strings"
"github.com/hpcloud/tail" "github.com/hpcloud/tail"
@ -100,7 +101,7 @@ type DefaultLogParser struct{}
func (p *DefaultLogParser) Parse(line string, toolName string) LogEntry { func (p *DefaultLogParser) Parse(line string, toolName string) LogEntry {
entry := NewLogEntry("log_entry") entry := NewLogEntry("log_entry")
entry.Tool = toolName entry.Tool = toolName
entry.Message = strings.TrimSpace(line) entry.LogMessage = strings.TrimSpace(line)
entry.Raw = line entry.Raw = line
return entry return entry
} }
@ -119,7 +120,7 @@ func (p *RegexLogParser) Parse(line string, toolName string) LogEntry {
if fields != nil { if fields != nil {
entry.Fields = fields entry.Fields = fields
} else { } else {
entry.Message = strings.TrimSpace(line) entry.LogMessage = strings.TrimSpace(line)
} }
return entry return entry
@ -159,34 +160,36 @@ func (p *NginxTJMLogParser) Parse(line string, toolName string) LogEntry {
entry := NewLogEntry("log_entry") entry := NewLogEntry("log_entry")
entry.Tool = toolName entry.Tool = toolName
entry.Raw = line entry.Raw = line
entry.Fields = p.parseNginxTJM(line) entry = p.parseNginxTJM(entry)
return entry return entry
} }
func (p *NginxTJMLogParser) parseNginxTJM(text string) map[string]any { func (p *NginxTJMLogParser) parseNginxTJM(entry LogEntry) LogEntry {
parts := strings.Fields(text) newEntry := entry
var nginxBase NGinXBaseInfo
parts := strings.Fields(entry.Raw)
if len(parts) < 10 { if len(parts) < 10 {
return map[string]any{ return newEntry
"raw": text,
}
} }
fields := make(map[string]any)
if len(parts) > 0 { if len(parts) > 0 {
timestamp := strings.Trim(parts[0], "[]") timestampStr := strings.Trim(parts[0], "[]")
fields["timestamp"] = timestamp timestamp, err := parseRFC3339WithOptionalZ(timestampStr)
if err != nil {
slog.Error("unable to parse time", "error", err)
}
newEntry.Timestamp = timestamp
} }
if len(parts) > 2 { if len(parts) > 2 {
fields["client_ip"] = parts[2] nginxBase.ClientIP = parts[2]
} }
for i, part := range parts { for i, part := range parts {
if strings.HasPrefix(part, "\"") { if strings.HasPrefix(part, "\"") {
if i+1 < len(parts) { if i+1 < len(parts) {
fields["method"] = strings.Trim(part, "\"") nginxBase.HTTPMethod = strings.Trim(part, "\"")
fields["route"] = parts[i+1] nginxBase.Route = parts[i+1]
} }
break break
} }
@ -194,10 +197,15 @@ func (p *NginxTJMLogParser) parseNginxTJM(text string) map[string]any {
for _, part := range parts { for _, part := range parts {
if after, ok := strings.CutPrefix(part, "status="); ok { if after, ok := strings.CutPrefix(part, "status="); ok {
fields["status"] = after statusCode, err := strconv.ParseInt(after, 10, 64)
if err != nil {
slog.Error("cant convert statuscode", "error", err)
}
nginxBase.StatusCode = int(statusCode)
break break
} }
} }
newEntry.BaseInformation = nginxBase
return fields return newEntry
} }

154
models.go
View file

@ -113,19 +113,51 @@ type NetworkStat struct {
} }
type LogEntry struct { type LogEntry struct {
Timestamp time.Time `json:"@timestamp"` Service string `json:"service,omitempty"`
Type string `json:"type"` Timestamp time.Time `json:"timestamp"`
Host string `json:"host"` Type string `json:"type"`
Tool string `json:"tool,omitempty"` Host string `json:"host"`
Service string `json:"service,omitempty"` Tool string `json:"tool,omitempty"`
Message string `json:"message,omitempty"` LogLevel string `json:"log_level"`
Fields map[string]any `json:"fields,omitempty"` LogMessage string `json:"message,omitempty"`
Raw string `json:"raw,omitempty"` SyslogInfo SyslogFields `json:"syslog_information,omitempty"`
Priority string `json:"priority,omitempty"` BaseInformation any `json:"base_info"`
Unit string `json:"unit,omitempty"` ServiceInformation any `json:"service_info"`
PID int `json:"pid,omitempty"` SystemMetrics SystemResources `json:"system-metrics"`
BootID string `json:"boot_id,omitempty"` ToolInformation any `json:"tool_info"`
MachineID string `json:"machine_id,omitempty"` Raw string `json:"raw,omitempty"`
Priority string `json:"priority,omitempty"`
PriorityName string `json:"priority_name,omitempty"`
Unit string `json:"unit,omitempty"`
PID int `json:"pid,omitempty"`
BootID string `json:"boot_id,omitempty"`
MachineID string `json:"machine_id,omitempty"`
Fields map[string]any `json:"fields,omitempty"`
}
type LogMessage struct {
Service string `json:"service"`
Timestamp time.Time `json:"timestamp"`
LogLevel string `json:"log_level"`
LogMessage string `json:"log_message"`
SyslogInfo SyslogFields `json:"syslog_information"`
BaseInformation any `json:"base_info"`
ServiceInformation any `json:"service_info"`
SystemMetrics SystemResources `json:"system-metrics"`
ToolInformation any `json:"tool_info"`
}
type SyslogFields struct {
SysLogTimestamp time.Time `json:"syslog_timestamp"`
Hostname string `json:"hostname"`
ProcessInfo string `json:"process_info"`
Raw string `json:"raw,omitempty"`
Priority string `json:"priority,omitempty"`
Unit string `json:"unit,omitempty"`
PID int `json:"pid,omitempty"`
BootID string `json:"boot_id,omitempty"`
MachineID string `json:"machine_id,omitempty"`
Fields map[string]any `json:"fields"`
} }
func NewLogEntry(entryType string) LogEntry { func NewLogEntry(entryType string) LogEntry {
@ -133,7 +165,7 @@ func NewLogEntry(entryType string) LogEntry {
Timestamp: time.Now(), Timestamp: time.Now(),
Type: entryType, Type: entryType,
Host: hostname, Host: hostname,
Fields: make(map[string]any), // Fields: make(map[string]any),
} }
} }
@ -149,3 +181,97 @@ func NewSystemResources() SystemResources {
BandwidthUtilization: make(map[string]BandwidthInfo), BandwidthUtilization: make(map[string]BandwidthInfo),
} }
} }
type AMBaseInfo struct {
ProcessID string `json:"process_id"`
ThreadID string `json:"thread_id"`
LoggerName string `json:"logger_name"`
}
type TCCBaseInfo struct {
ProcessID string `json:"process_id"`
ThreadID string `json:"thread_id"`
LoggerName string `json:"logger_name"`
}
type NGinXBaseInfo struct {
ClientIP string `json:"client_ip"`
RemoteUser string `json:"remote_user"`
Request string `json:"request"`
StatusCode int `json:"status_code"`
BytesSend int `json:"bytes_sent"`
Referer string `json:"referer"`
UserAgent string `json:"user_agent"`
HTTPMethod string `json:"http_method"`
RequestURI string `json:"request_uri"`
HTTPVersion string `json:"http_version"`
Route string `json:"route"`
}
type TSBaseInfo struct {
TransferID string `json:"transfer_identifier"`
Lane int `json:"thread"`
Direction string `json:"direction"`
Buffers int `json:"buffers"`
FileCount int `json:"file_count"`
FileSize float64 `json:"file_size"`
ChunkSize int `json:"chunksize"`
Streams int `json:"streams"`
TargetDatarate float64 `json:"target_datarate"`
Protocoll string `json:"protocoll"`
Destination string `json:"destination"`
Sender string `json:"sender_id"`
Target string `json:"transfer_target"`
Source string `json:"source"`
Receiver string `json:"receiver_id"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
}
type TSTransferInfo struct {
TransferID string `json:"transfer_identifier"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
TransferLane string `json:"transfer_lane"`
FileCount int `json:"file_count"`
FileSizeMB float64 `json:"file_size_mb"`
DataRateMBs float64 `json:"datarate_mbs"`
Dest string `json:"destination"`
Src string `json:"source"`
SenderID string `json:"sender_id"`
Receiver string `json:"receiver"`
BytesProcessed int64 `json:"bytes_processed"`
Direction string `json:"direction"`
Duration time.Duration `json:"duration_seconds"`
TheoreticalRate float64 `json:"theoretical_rate_mbs"`
Efficiency float64 `json:"efficiency_percent"`
Status string `json:"status"`
}
type TJMBaseInfo struct {
ProcessID string `json:"process_id"`
TransferID string `json:"transfer_identifier"`
Direction string `json:"direction"`
Username string `json:"username"`
CorrelationID string `json:"correlation_id"`
ThreadID string `json:"thread_id"`
JavaClass string `json:"java_class"`
}
type TJMTransferInfo struct {
TransferID string `json:"transfer_identifier"`
TransferName string `json:"transfer_name"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
TransferLane string `json:"transfer_lane"`
Dest string `json:"destination"`
Src string `json:"source"`
SenderID string `json:"sender_id"`
Receiver string `json:"receiver"`
Direction string `json:"direction"`
Worker string `json:"worker"`
Duration time.Duration `json:"duration"`
DataRateMBs float64 `json:"datarate_mbs"`
BytesProcessed int64 `json:"bytes_processed"`
FileSizeMB float64 `json:"file_size_mb"`
}

304
service_monitor.go
View file

@ -136,17 +136,18 @@ func (jep *JournalEntryParser) Parse(jsonLine string) (LogEntry, error) {
} }
if msg, ok := journalData["MESSAGE"].(string); ok { if msg, ok := journalData["MESSAGE"].(string); ok {
entry.Message = msg entry.LogMessage = msg
} }
if priority, ok := journalData["PRIORITY"].(string); ok { if priority, ok := journalData["PRIORITY"].(string); ok {
entry.Priority = priority entry.Priority = priority
entry.Fields["priority_name"] = jep.getPriorityName(priority) entry.PriorityName = jep.getPriorityName(priority)
} }
if pidStr, ok := journalData["_PID"].(string); ok { if pidStr, ok := journalData["_PID"].(string); ok {
if pid, err := strconv.Atoi(pidStr); err == nil { if pid, err := strconv.Atoi(pidStr); err == nil {
entry.PID = pid entry.PID = pid
entry.SyslogInfo.ProcessInfo = strconv.FormatInt(int64(pid), 10)
} }
} }
@ -197,7 +198,7 @@ func (jep *JournalEntryParser) extractSystemdFields(journalData map[string]any,
for _, field := range systemdFields { for _, field := range systemdFields {
if value, ok := journalData[field]; ok { if value, ok := journalData[field]; ok {
esFieldName := strings.ToLower(strings.TrimPrefix(field, "_")) esFieldName := strings.ToLower(strings.TrimPrefix(field, "_"))
entry.Fields[esFieldName] = value entry.SyslogInfo.Fields[esFieldName] = value
} }
} }
} }
@ -237,115 +238,114 @@ var (
func parseTixstreamService(entry LogEntry) LogEntry { func parseTixstreamService(entry LogEntry) LogEntry {
newEntry := entry newEntry := entry
fields := make(map[string]any) var baseInfo TSBaseInfo
matches := tsServicePattern.FindStringSubmatch(newEntry.Message) matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
if len(matches) > 0 { if len(matches) > 0 {
timestamp := strings.Join(strings.Split(matches[2], " "), "T") timestamp := strings.Join(strings.Split(matches[2], " "), "T")
fields["log_level"] = strings.TrimSpace(matches[1]) newEntry.LogLevel = strings.TrimSpace(matches[1])
fields["message_timestamp"] = timestamp if newEntry.Timestamp.IsZero() {
fields["log_message"] = strings.TrimSpace(matches[3]) timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
} else { if err != nil {
fields["log_message"] = newEntry.Message slog.Error("cant parse time string", "error", err)
}
newEntry.Timestamp = timeParsed
}
newEntry.LogMessage = strings.TrimSpace(matches[3])
} }
trNameMatch := tsTransferIDPattern.FindStringSubmatch(fields["log_message"].(string)) trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
var transferID string var transferID string
if len(trNameMatch) > 0 { if len(trNameMatch) > 0 {
transferID = trNameMatch[1] transferID = trNameMatch[1]
fields["log_message"] = trNameMatch[2] newEntry.LogMessage = trNameMatch[2]
split := strings.Fields(trNameMatch[2]) split := strings.Fields(trNameMatch[2])
switch split[0] { switch split[0] {
case "in:": case "in:":
fields["transfer_direction"] = "incoming" baseInfo.Direction = "incoming"
case "out:": case "out:":
fields["transfer_direction"] = "outgoing" baseInfo.Direction = "outgoing"
} }
} }
msg := strings.ReplaceAll(newEntry.Message, " ", " ") msg := strings.ReplaceAll(newEntry.LogMessage, " ", " ")
parts := strings.Fields(msg) parts := strings.Fields(msg)
if len(parts) < 5 { if len(parts) < 5 {
return newEntry return newEntry
} }
tsDetail := tsDetailPattern1.FindStringSubmatch(fields["log_message"].(string)) tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0]) threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
fields["thread"] = threadInt
buffersInt, _ := strconv.Atoi(tsDetail[2]) buffersInt, _ := strconv.Atoi(tsDetail[2])
fields["buffers"] = buffersInt
fileCountInt, _ := strconv.Atoi(tsDetail[3]) fileCountInt, _ := strconv.Atoi(tsDetail[3])
fields["file_count"] = fileCountInt
fileSizeInt, _ := strconv.Atoi(tsDetail[4])
fields["file_size"] = fileSizeInt
chunkSizeInt, _ := strconv.Atoi(tsDetail[5]) chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
fields["chunksize"] = chunkSizeInt
streamsInt, _ := strconv.Atoi(tsDetail[6]) streamsInt, _ := strconv.Atoi(tsDetail[6])
fields["streams"] = streamsInt
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64) datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
fields["target_datarate"] = datarateFloat fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
fields["protocoll"] = tsDetail[8] baseInfo.Lane = threadInt
fields["destination"] = tsDetail[9] baseInfo.Buffers = buffersInt
fields["sender_id"] = tsDetail[10] baseInfo.FileCount = fileCountInt
baseInfo.FileSize = fileSizeFloat
baseInfo.ChunkSize = chunkSizeInt
baseInfo.Streams = streamsInt
baseInfo.TargetDatarate = datarateFloat
baseInfo.Protocoll = tsDetail[8]
baseInfo.Destination = tsDetail[9]
baseInfo.Sender = tsDetail[10]
} }
tsDetail = tsDetailPattern2.FindStringSubmatch(fields["log_message"].(string)) tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { if len(tsDetail) > 0 {
fields["transfer_target"] = tsDetail[1] baseInfo.Target = tsDetail[1]
} }
tsDetail = tsDetailPattern3.FindStringSubmatch(fields["log_message"].(string)) tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0]) threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
fields["thread"] = threadInt
buffersInt, _ := strconv.Atoi(tsDetail[2]) buffersInt, _ := strconv.Atoi(tsDetail[2])
fields["buffers"] = buffersInt
fileCountInt, _ := strconv.Atoi(tsDetail[3]) fileCountInt, _ := strconv.Atoi(tsDetail[3])
fields["file_count"] = fileCountInt fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
fileSizeInt, _ := strconv.Atoi(tsDetail[4])
fields["file_size"] = fileSizeInt
chunkSizeInt, _ := strconv.Atoi(tsDetail[5]) chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
fields["chunksize"] = chunkSizeInt
streamsInt, _ := strconv.Atoi(tsDetail[6]) streamsInt, _ := strconv.Atoi(tsDetail[6])
fields["streams"] = streamsInt
datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64) datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
fields["target_datarate"] = datarateFloat baseInfo.Lane = threadInt
fields["protocoll"] = tsDetail[8] baseInfo.Buffers = buffersInt
fields["source"] = tsDetail[9] baseInfo.FileCount = fileCountInt
fields["receiver_id"] = tsDetail[10] baseInfo.FileSize = fileSizeFloat
baseInfo.ChunkSize = chunkSizeInt
baseInfo.Streams = streamsInt
baseInfo.TargetDatarate = datarateFloat
baseInfo.Protocoll = tsDetail[8]
baseInfo.Source = tsDetail[9]
baseInfo.Receiver = tsDetail[10]
} }
tsDetail = tsDetailPattern4.FindStringSubmatch(fields["log_message"].(string)) tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
if len(tsDetail) > 0 { if len(tsDetail) > 0 {
threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0]) threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
fields["thread"] = threadInt baseInfo.Lane = threadInt
fields["source"] = tsDetail[2] baseInfo.Source = tsDetail[2]
fields["destination"] = tsDetail[3] baseInfo.Destination = tsDetail[3]
fields["item"] = tsDetail[4]
fields["count"] = tsDetail[5]
} }
if strings.Contains(fields["log_message"].(string), "Transfer start") || strings.Contains(fields["log_message"].(string), "Transfer started,") { if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
fields["event"] = "transfer_started" baseInfo.StartTime = newEntry.Timestamp
fields["start_time"] = fields["message_timestamp"]
} }
if strings.Contains(fields["log_message"].(string), "Transfer stopped local state=finished") { if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
fields["event"] = "transfer_stopped" baseInfo.EndTime = newEntry.Timestamp
fields["end_time"] = fields["message_timestamp"]
} }
// value, ok := fields["transfer_id"]
if transferID != "" { if transferID != "" {
fields["transfer_identifier"] = transferID baseInfo.TransferID = transferID
} else {
baseInfo.TransferID = "no_transfer_id"
} }
if fields["transfer_identifier"] == nil { if !baseInfo.StartTime.IsZero() {
fields["transfer_identifier"] = "unknown" newEntry.BaseInformation = baseInfo
} }
if fields["message_timestamp"] == nil {
fields["message_timestamp"] = newEntry.Timestamp
}
newEntry.Fields = fields
return newEntry return newEntry
} }
func parseTJMService(entry LogEntry) LogEntry { func parseTJMService(entry LogEntry) LogEntry {
newEntry := entry newEntry := entry
logContent := entry.Message var baseInfo TJMBaseInfo
logContent := entry.LogMessage
msg := strings.TrimSpace(logContent) msg := strings.TrimSpace(logContent)
msg = strings.ReplaceAll(msg, " ", " ") msg = strings.ReplaceAll(msg, " ", " ")
msg = strings.ReplaceAll(msg, "---", "") msg = strings.ReplaceAll(msg, "---", "")
@ -354,156 +354,160 @@ func parseTJMService(entry LogEntry) LogEntry {
if len(parts) < 4 { if len(parts) < 4 {
return newEntry return newEntry
} }
fields := make(map[string]any)
matches := tjmServicePattern.FindStringSubmatch(logContent) matches := tjmServicePattern.FindStringSubmatch(logContent)
if len(matches) > 0 { if len(matches) > 0 {
timestamp := strings.Join(strings.Split(matches[1], " "), "T") timestamp := strings.Join(strings.Split(matches[2], " "), "T")
fields["message_timestamp"] = timestamp newEntry.LogLevel = strings.TrimSpace(matches[1])
fields["log_level"] = strings.TrimSpace(matches[2]) if newEntry.Timestamp.IsZero() {
fields["pid"] = strings.TrimSpace(matches[3]) timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
fields["correlation_id"] = strings.TrimSpace(matches[4]) if err != nil {
fields["username"] = strings.TrimSpace(matches[5]) slog.Error("cant parse time string", "error", err)
fields["thread_id"] = strings.TrimSpace(matches[6]) }
fields["java_class"] = strings.TrimSpace(matches[7]) newEntry.Timestamp = timeParsed
fields["log_message"] = strings.TrimSpace(matches[8]) }
newEntry.LogLevel = strings.TrimSpace(matches[2])
newEntry.LogMessage = strings.TrimSpace(matches[8])
baseInfo = TJMBaseInfo{
ProcessID: strings.TrimSpace(matches[3]),
CorrelationID: strings.TrimSpace(matches[4]),
Username: strings.TrimSpace(matches[5]),
ThreadID: strings.TrimSpace(matches[6]),
JavaClass: strings.TrimSpace(matches[7]),
}
} else { } else {
fields["log_message"] = logContent newEntry.LogMessage = logContent
} }
trNameMatch := tjmTransferNamePattern.FindStringSubmatch(fields["log_message"].(string)) trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
var transferName string var transferName string
var transferID string var transferID string
if len(trNameMatch) > 0 { if len(trNameMatch) > 0 {
transferName = trNameMatch[1] transferName = trNameMatch[1]
fields["log_message"] = trNameMatch[2] newEntry.LogMessage = trNameMatch[2]
if strings.Contains(trNameMatch[1], "-in") { if strings.Contains(trNameMatch[1], "-in") {
fields["transfer_direction"] = "incoming" baseInfo.Direction = "incoming"
} }
if strings.Contains(trNameMatch[1], "-out") { if strings.Contains(trNameMatch[1], "-out") {
fields["transfer_direction"] = "outgoing" baseInfo.Direction = "outgoing"
} }
} }
trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(fields["log_message"].(string)) trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
if len(trIDMatch) > 0 { if len(trIDMatch) > 0 {
transferID = trIDMatch[1] transferID = trIDMatch[1]
} }
trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(fields["log_message"].(string)) trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
if len(trIDMatch) > 0 { if len(trIDMatch) > 0 {
transferID = trIDMatch[2] transferID = trIDMatch[2]
} }
// value, ok := fields["transfer_id"]
if transferID != "" { if transferID != "" {
fields["transfer_identifier"] = transferID baseInfo.TransferID = transferID
} else if transferName != "" { } else if transferName != "" {
// value, ok := fields["transfer_name"] baseInfo.TransferID = transferName
// if ok { } else {
fields["transfer_identifier"] = transferName baseInfo.TransferID = "no_transfer_id"
// }
} }
if fields["transfer_identifier"] == nil { newEntry.BaseInformation = baseInfo
fields["transfer_identifier"] = "unknown"
}
if fields["message_timestamp"] == nil {
fields["message_timestamp"] = newEntry.Timestamp
}
newEntry.Fields = fields
return newEntry return newEntry
} }
func parseAMService(entry LogEntry) LogEntry { func parseAMService(entry LogEntry) LogEntry {
newEntry := entry newEntry := entry
if newEntry.Fields == nil { logContent := newEntry.LogMessage
newEntry.Fields = make(map[string]any)
}
fields := make(map[string]any)
matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(entry.Message)) matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
if len(matches) != 7 { if len(matches) != 7 {
return newEntry return newEntry
} }
timestamp := strings.Join(strings.Split(matches[1], " "), "T") timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
fields["message_timestamp"] = timestamp if newEntry.Timestamp.IsZero() {
fields["log_level"] = matches[2] timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
fields["process_id"] = matches[3] if err != nil {
fields["thread_id"] = strings.TrimSpace(matches[4]) slog.Error("cant parse time string", "error", err)
fields["logger_name"] = matches[5] }
fields["log_message"] = matches[6] newEntry.Timestamp = timeParsed
fields["transfer_identifier"] = "unknown"
if fields["message_timestamp"] == nil {
fields["message_timestamp"] = newEntry.Timestamp
} }
newEntry.Fields = fields baseInfo := AMBaseInfo{
ProcessID: matches[3],
ThreadID: strings.TrimSpace(matches[4]),
LoggerName: matches[5],
}
newEntry.LogLevel = matches[2]
newEntry.LogMessage = matches[6]
newEntry.BaseInformation = baseInfo
return newEntry return newEntry
} }
func parseTCCService(entry LogEntry) LogEntry { func parseTCCService(entry LogEntry) LogEntry {
newEntry := entry newEntry := entry
if newEntry.Fields == nil { logContent := newEntry.LogMessage
newEntry.Fields = make(map[string]any)
}
fields := make(map[string]any) matches := tccServicePattern.FindStringSubmatch(logContent)
matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(entry.Message))
if len(matches) != 7 { if len(matches) != 7 {
return newEntry return newEntry
} }
timestamp := strings.Join(strings.Split(matches[1], " "), "T") timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
fields["message_timestamp"] = timestamp if newEntry.Timestamp.IsZero() {
fields["log_level"] = matches[2] timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
fields["process_id"] = matches[3] if err != nil {
fields["thread_id"] = strings.TrimSpace(matches[4]) slog.Error("cant parse time string", "error", err)
fields["logger_name"] = matches[5] }
fields["log_message"] = matches[6] newEntry.Timestamp = timeParsed
fields["transfer_identifier"] = "unknown"
if fields["message_timestamp"].(string) == "" {
fields["message_timestamp"] = newEntry.Timestamp
} }
newEntry.Fields = fields baseInfo := TCCBaseInfo{
ProcessID: matches[3],
newEntry.Fields["timestamp"] = matches[1] ThreadID: strings.TrimSpace(matches[4]),
newEntry.Fields["log_level"] = matches[2] LoggerName: matches[5],
newEntry.Fields["process_id"] = matches[3] }
newEntry.Fields["thread_name"] = strings.TrimSpace(matches[4]) newEntry.LogLevel = matches[2]
newEntry.Fields["logger_name"] = matches[5] newEntry.LogMessage = matches[6]
newEntry.Fields["log_message"] = matches[6] newEntry.BaseInformation = baseInfo
return newEntry return newEntry
} }
func parseNginxService(entry LogEntry) LogEntry { func parseNginxService(entry LogEntry) LogEntry {
newEntry := entry newEntry := entry
if newEntry.Fields == nil {
newEntry.Fields = make(map[string]any)
}
matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.Message)) matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
if len(matches) < 7 { if len(matches) < 7 {
return newEntry return newEntry
} }
statusCode, err := strconv.ParseInt(matches[5], 10, 64)
newEntry.Fields["client_ip"] = matches[1] if err != nil {
newEntry.Fields["remote_user"] = matches[2] slog.Error("cant parse statuscode", "error", err)
newEntry.Fields["timestamp"] = matches[3] }
newEntry.Fields["request"] = matches[4] bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
newEntry.Fields["status_code"] = matches[5] if err != nil {
newEntry.Fields["bytes_sent"] = matches[6] slog.Error("cant parse bytessend", "error", err)
}
baseInfo := NGinXBaseInfo{
ClientIP: matches[1],
RemoteUser: matches[2],
Request: matches[4],
StatusCode: int(statusCode),
BytesSend: int(bytesSend),
}
if len(matches) > 7 && matches[7] != "" { if len(matches) > 7 && matches[7] != "" {
newEntry.Fields["referer"] = matches[7] baseInfo.Referer = matches[7]
} }
if len(matches) > 8 && matches[8] != "" { if len(matches) > 8 && matches[8] != "" {
newEntry.Fields["user_agent"] = matches[8] baseInfo.UserAgent = matches[8]
} }
if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 { if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
newEntry.Fields["http_method"] = requestParts[0] baseInfo.HTTPMethod = requestParts[0]
newEntry.Fields["request_uri"] = requestParts[1] baseInfo.RequestURI = requestParts[1]
newEntry.Fields["http_version"] = requestParts[2] baseInfo.HTTPVersion = requestParts[2]
} }
return newEntry return newEntry
} }
func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
timeStr += "Z"
}
return time.Parse(time.RFC3339Nano, timeStr)
}

12
system_metrics.go
View file

@ -389,18 +389,6 @@ func (smc *SystemMetricsCollector) collectSystemLimits(result *SystemResources)
return nil return nil
} }
// // Hilfsfunktionen
// func NewSystemResources() SystemResources {
// return SystemResources{
// Timestamp: time.Now(),
// DiskUsage: make(map[string]DiskUsage),
// DiskIOStats: make(map[string]DiskIOStat),
// NetworkStats: make(map[string]NetworkStat),
// NetworkLatency: make(map[string]LatencyInfo),
// BandwidthUtilization: make(map[string]BandwidthInfo),
// }
// }
func (smc *SystemMetricsCollector) collectProcessMetrics(result *SystemResources) error { func (smc *SystemMetricsCollector) collectProcessMetrics(result *SystemResources) error {
processes, err := process.Processes() processes, err := process.Processes()
if err != nil { if err != nil {

19
web_service.go
View file

@ -77,14 +77,15 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
} }
size := ws.parseSizeParam(r) size := ws.parseSizeParam(r)
since := ws.parseSinceParam(r)
slog.Info("Export request received", "indices", indices, "size", size) slog.Info("Export request received", "indices", indices, "size", size, "since", since)
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Disposition", "attachment; filename=elasticsearch_export.json") w.Header().Set("Content-Disposition", "attachment; filename=elasticsearch_export.json")
exporter := NewElasticsearchExporter(ws.esClient) exporter := NewElasticsearchExporter(ws.esClient)
if err := exporter.ExportToStream(r.Context(), indices, size, w); err != nil { if err := exporter.ExportToStream(r.Context(), indices, size, since, w); err != nil {
slog.Error("export error", "error", err) slog.Error("export error", "error", err)
http.Error(w, fmt.Sprintf("Export error: %v", err), http.StatusInternalServerError) http.Error(w, fmt.Sprintf("Export error: %v", err), http.StatusInternalServerError)
return return
@ -204,6 +205,20 @@ func (ws *WebService) parseIndicesParam(r *http.Request) []string {
return result return result
} }
func (ws *WebService) parseSinceParam(r *http.Request) int {
sinceParam := r.URL.Query().Get("since")
if sinceParam == "" {
return 0
}
since, err := strconv.Atoi(sinceParam)
if err != nil {
return 0
}
return since
}
func (ws *WebService) parseSizeParam(r *http.Request) int { func (ws *WebService) parseSizeParam(r *http.Request) int {
sizeParam := r.URL.Query().Get("size") sizeParam := r.URL.Query().Get("size")
if sizeParam == "" { if sizeParam == "" {