diff --git a/config.go b/config.go index 93f0a88..6527410 100644 --- a/config.go +++ b/config.go @@ -2,8 +2,8 @@ package main import ( "fmt" - "log" "log/slog" + "regexp" "time" "github.com/spf13/viper" @@ -96,6 +96,7 @@ type Config struct { Level string `mapstructure:"level"` FilePath string `mapstructure:"file_path"` } `mapstructure:"logging"` + PatternsFile string `mapstructure:"patterns_file"` } type StorageRotationConfig struct { @@ -125,30 +126,6 @@ func (src StorageRotationConfig) GetCheckInterval() time.Duration { return time.Duration(src.CheckIntervalMinutes) * time.Minute } -func LoadConfig() (*Config, error) { - viper.SetConfigName("config") - viper.AddConfigPath(".") - viper.AddConfigPath("/opt/tixel/tixel-watch/") - viper.SetConfigType("yaml") - - setConfigDefaults() - - if err := viper.ReadInConfig(); err != nil { - return nil, fmt.Errorf("error reading config: %w", err) - } - - var cfg Config - if err := viper.Unmarshal(&cfg); err != nil { - return nil, fmt.Errorf("error parsing config: %w", err) - } - - if err := validateConfig(&cfg); err != nil { - return nil, fmt.Errorf("config validation failed: %w", err) - } - - return &cfg, nil -} - func setConfigDefaults() { viper.SetDefault("poll_interval_seconds", 30) viper.SetDefault("elasticsearch.timeout", 30) @@ -162,21 +139,6 @@ func setConfigDefaults() { viper.SetDefault("web_service.port", 8080) viper.SetDefault("web_service.host", "localhost") viper.SetDefault("logging.level", "info") -} - -func setConfigDefaultsV2() { - viper.SetDefault("poll_interval_seconds", 30) - viper.SetDefault("elasticsearch.timeout", 30) - viper.SetDefault("system_metrics.enabled", true) - viper.SetDefault("system_metrics.collect_cpu", true) - viper.SetDefault("system_metrics.collect_memory", true) - viper.SetDefault("system_metrics.collect_disk", true) - viper.SetDefault("system_metrics.collect_network", false) - viper.SetDefault("system_metrics.disk_paths", []string{"/"}) - viper.SetDefault("web_service.enabled", false) - viper.SetDefault("web_service.port", 8080) - viper.SetDefault("web_service.host", "localhost") - viper.SetDefault("logging.level", "info") viper.SetDefault("export.enabled", true) viper.SetDefault("export.batch_size", 100) viper.SetDefault("export.export_interval", "30s") @@ -190,38 +152,16 @@ func setConfigDefaultsV2() { viper.SetDefault("localstorage.rotation.max_files", 7) viper.SetDefault("localstorage.rotation.check_interval_minutes", 5) viper.SetDefault("localstorage.rotation.archive_dir", "") + viper.SetDefault("patterns_file", "./configs/patterns.yaml") } -func validateConfig(cfg *Config) error { - if cfg.Elasticsearch.URL == "" { - return fmt.Errorf("elasticsearch.url is required") - } - - if cfg.Elasticsearch.Index == "" { - return fmt.Errorf("elasticsearch.index is required") - } - - if cfg.PollIntervalSeconds <= 0 { - log.Printf("Warning: poll_interval_seconds is %d, setting to 30", cfg.PollIntervalSeconds) - cfg.PollIntervalSeconds = 30 - } - - for i := range cfg.Tools { - if cfg.Tools[i].BufferSize <= 0 { - cfg.Tools[i].BufferSize = 100 - } - } - - return nil -} - -func LoadConfigV2() (*Config, error) { +func LoadConfig() (*Config, error) { viper.SetConfigName("config") viper.AddConfigPath(".") viper.AddConfigPath("/opt/tixel/tixel-watch/") viper.SetConfigType("yaml") - setConfigDefaultsV2() + setConfigDefaults() if err := viper.ReadInConfig(); err != nil { return nil, fmt.Errorf("error reading config: %w", err) @@ -232,14 +172,14 @@ func LoadConfigV2() (*Config, error) { return nil, fmt.Errorf("error parsing config: %w", err) } - if err := validateConfigV2(&cfg); err != nil { + if err := validateConfig(&cfg); err != nil { return nil, fmt.Errorf("config validation failed: %w", err) } return &cfg, nil } -func validateConfigV2(cfg *Config) error { +func validateConfig(cfg *Config) error { if !cfg.LocalStorage.Enable { return fmt.Errorf("local storage must be enabled in the new architecture") } @@ -280,9 +220,15 @@ func validateConfigV2(cfg *Config) error { } } - for i := range cfg.Tools { - if cfg.Tools[i].BufferSize <= 0 { - cfg.Tools[i].BufferSize = 100 + for _, tool := range cfg.Tools { + if tool.BufferSize <= 0 { + tool.BufferSize = 100 + } + + if tool.Format.Pattern != "" { + if _, err := regexp.Compile(tool.Format.Pattern); err != nil { + return fmt.Errorf("invalid regex for tool '%s': %w", tool.Name, err) + } } } diff --git a/configs/patterns.yml b/configs/patterns.yml new file mode 100644 index 0000000..069a38c --- /dev/null +++ b/configs/patterns.yml @@ -0,0 +1,164 @@ +patterns: + # =========================================================================== + # Common / Shared Patterns + # =========================================================================== + common: + extractors: + - name: "syslog_header" + regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P[^\s]+) (?P[^:]+):\s*(?P.*)$' + fields: + syslog_timestamp: "time:Jan 02 15:04:05" + hostname: "string" + process_info: "string" + message_rest: "string" + + - name: "timestamp_rfc3339" + regex: '(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)' + fields: + timestamp: "time:2006-01-02T15:04:05.000000Z" + + # =========================================================================== + # TIXstream Service + # Deckt ab: tsServicePattern, tsTransferIDPattern, tsDetailPattern1-4 + # =========================================================================== + tixstream: + extractors: + - name: "service_log_base" + regex: '^(?P\S+)\s+(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?P.*)' + fields: + log_level: "string" + timestamp: "time:2006-01-02 15:04:05.000000" + message: "string" + + - name: "transfer_id_extraction" + regex: '^(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?P.*)' + fields: + transfer_id: "string" + message: "string" + + - name: "transfer_start_in" + regex: 'in: Transfer start (?P\d+/\d+) buffers=(?P\d+) files=(?P\d+) size=(?P[0-9.]+) MByte chunksize=(?P\d+) streams=(?P\d+) target-datarate=(?P[0-9.]+) MByte/s protocol=(?P\w+) dest=(?P\S+) sender-id=(?P\S+)' + fields: + thread_info: "string" # z.B. "1/4" - Typisierung hier schwierig, also String + buffers: "int" + file_count: "int" + size_mb: "float" + chunk_size: "int" + streams: "int" + target_rate: "float" + protocol: "string" + destination: "string" + sender_id: "string" + direction: "string" # Wir können statische Felder im Parser injecten oder hier als "implizit" betrachten + + - name: "transfer_start_remote_out" + regex: 'out: Start remote transfer to (?P[^\s]+) request executed, duration=(?P[0-9.]+) s' + fields: + target: "string" + duration: "float" + + - name: "transfer_start_out" + regex: 'out: Transfer start (?P\d+/\d+) buffers=(?P\d+) files=(?P\d+) size=(?P[0-9.]+) MByte chunksize=(?P\d+) streams=(?P\d+) target-datarate=(?P[0-9.]+) MByte/s protocol=(?P\w+) src=(?P\S+) receiver=(?P\S+)' + fields: + thread_info: "string" + buffers: "int" + file_count: "int" + size_mb: "float" + chunk_size: "int" + streams: "int" + target_rate: "float" + protocol: "string" + source: "string" + receiver: "string" + + - name: "transfer_start_generic" + regex: 'out: Start transfer (?P\d+/\d+), src=(?P[^ ]*) dest=(?P[^ ]*) item\[0\]=(?P[^ ]*) count=(?P\d+)' + fields: + thread_info: "string" + source: "string" + destination: "string" + item0: "string" + count: "int" + + # =========================================================================== + # Transfer Job Manager (TJM) + # Deckt ab: tjmServicePattern, tjmTransferNamePattern, tjmTransferIDPattern1/2 + # =========================================================================== + transfer-job-manager: + extractors: + - name: "service_log_base" + regex: '^(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?P\S+)\s+(?P\d+).*?\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+(?P.*?)\s+:\s+(?P.*)' + fields: + timestamp: "time:2006-01-02 15:04:05.000" + log_level: "string" + pid: "int" + correlation_id: "string" + username: "string" + thread_id: "string" + java_class: "string" + message: "string" + + - name: "transfer_name_info" + regex: '^(?P\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (?P.*)$' + fields: + transfer_name_raw: "string" + message: "string" + + - name: "transfer_id_mid" + regex: '(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P.*)' + fields: + transfer_id: "string" + message: "string" + + - name: "transfer_id_prefixed" + regex: '(?P.*)(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P.*)' + fields: + prefix: "string" + transfer_id: "string" + message: "string" + + # =========================================================================== + # Access Manager & TCC + # Deckt ab: amServicePattern, tccServicePattern + # =========================================================================== + access-manager: + extractors: + - name: "spring_boot_log" + regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P\w+)\s+(?P\d+)\s+---\s+\[\s*(?P[^\]]*)\]\s+(?P[\w\.]+)\s*:\s+(?P.*)$' + fields: + timestamp: "time:2006-01-02T15:04:05.000000Z" + log_level: "string" + pid: "int" + thread_id: "string" + logger: "string" + message: "string" + + tixel-control-center: + extractors: + - name: "spring_boot_log" + regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P\w+)\s+(?P\d+)\s+---\s+\[\s*(?P[^\]]*)\]\s+(?P[\w\.]+)\s*:\s+(?P.*)$' + fields: + timestamp: "time:2006-01-02T15:04:05.000000Z" + log_level: "string" + pid: "int" + thread_id: "string" + logger: "string" + message: "string" + + # =========================================================================== + # Nginx + # Deckt ab: nginxAccessPattern + # =========================================================================== + nginx: + extractors: + - name: "access_log" + regex: '^(?P\S+)\s+\S+\s+(?P\S+)\s+\[(?P[^\]]+)\]\s+"(?P[^"]+)"\s+(?P\d+)\s+(?P\d+|-)\s*(?:"(?P[^"]*)"\s+"(?P[^"]*)")?' + fields: + client_ip: "string" + remote_user: "string" + timestamp_nginx: "string" + request: "string" + status_code: "int" + bytes_sent: "int" + referer: "string" + user_agent: "string" diff --git a/elasticsearch.go b/elasticsearch.go index 3b3537d..6b217c8 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -7,7 +7,7 @@ import ( "log/slog" "strings" "time" - "tixel_watch/models" + "watch-tool/models" "github.com/elastic/go-elasticsearch/v8" ) diff --git a/elasticsearch_exporter.go b/elasticsearch_exporter.go index 0a36f5f..55f4dbe 100644 --- a/elasticsearch_exporter.go +++ b/elasticsearch_exporter.go @@ -4,280 +4,87 @@ import ( "context" "encoding/json" "fmt" - "io" "log/slog" "strings" "time" + "watch-tool/models" "github.com/elastic/go-elasticsearch/v8" ) type ElasticsearchExporter struct { client *elasticsearch.Client + config ElasticsearchConfig } -func NewElasticsearchExporter(client *elasticsearch.Client) *ElasticsearchExporter { +func NewElasticsearchExporter(config ElasticsearchConfig) (*ElasticsearchExporter, error) { + client, err := NewElasticsearchClient(config) + if err != nil { + return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err) + } + return &ElasticsearchExporter{ client: client, - } + config: config, + }, nil } -type ExportResult struct { - Index string `json:"index"` - DocumentCount int `json:"document_count"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - Duration string `json:"duration"` - Error string `json:"error,omitempty"` -} - -func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []string, batchSize int, since int, writer io.Writer) error { - startTime := time.Now() - - if _, err := writer.Write([]byte("{\n \"export_info\": {\n")); err != nil { - return fmt.Errorf("error writing export header: %w", err) +func (e *ElasticsearchExporter) Export(ctx context.Context, entries []models.LogMessage) error { + if len(entries) == 0 { + return nil } - exportInfo := map[string]any{ - "timestamp": startTime, - "indices": indices, - "batch_size": batchSize, - "sinceDays": since, - } + var body strings.Builder + for _, entry := range entries { + indexName := e.config.Index - infoBytes, err := json.MarshalIndent(exportInfo, " ", " ") - if err != nil { - return fmt.Errorf("error marshalling export info: %w", err) - } + indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName) + body.WriteString(indexLine) + body.WriteString("\n") - infoStr := string(infoBytes) - infoStr = strings.TrimPrefix(infoStr, "{") - infoStr = strings.TrimSuffix(infoStr, "}") - - if _, err := writer.Write([]byte(infoStr)); err != nil { - return fmt.Errorf("error writing export info: %w", err) - } - - if _, err := writer.Write([]byte("\n },\n \"data\": {\n")); err != nil { - return fmt.Errorf("error writing data header: %w", err) - } - - results := make([]ExportResult, 0, len(indices)) - first := true - - for _, index := range indices { - if !first { - if _, err := writer.Write([]byte(",\n")); err != nil { - return fmt.Errorf("error writing separator: %w", err) - } - } - first = false - - result := e.exportIndex(ctx, index, batchSize, since, writer) - results = append(results, result) - - if result.Error != "" { - slog.Error("error exporting index", "index", index, "error", result.Error) + data, err := json.Marshal(entry) + if err != nil { + slog.Error("error marshalling JSON", "error", err) + continue } + body.WriteString(string(data)) + body.WriteString("\n") } - if _, err := writer.Write([]byte("\n },\n \"results\": ")); err != nil { - return fmt.Errorf("error writing results header: %w", err) - } + timeout := time.Duration(e.config.Timeout) * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() - if err := json.NewEncoder(writer).Encode(results); err != nil { - return fmt.Errorf("error writing results: %w", err) - } - - if _, err := writer.Write([]byte("}\n")); err != nil { - return fmt.Errorf("error writing final bracket: %w", err) - } - - duration := time.Since(startTime) - slog.Info("Export completed", "duration", duration, "indices_count", len(indices)) - - return nil -} - -func (e *ElasticsearchExporter) exportIndex(ctx context.Context, index string, batchSize int, since int, writer io.Writer) ExportResult { - startTime := time.Now() - result := ExportResult{ - Index: index, - StartTime: startTime, - } - - if _, err := fmt.Fprintf(writer, " \"%s\": [\n", index); err != nil { - result.Error = fmt.Sprintf("error writing index header: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - - query := `{"query":{"match_all":{}}}` - if since > 0 { - query = fmt.Sprintf(`{ - "query": { - "range": { - "timestamp": { - "gte": "now-%dd/d", - "lt": "now/d" - } - } - } -}`, since) - } - res, err := e.client.Search( - e.client.Search.WithContext(ctx), - e.client.Search.WithIndex(index), - e.client.Search.WithScroll(1000), - e.client.Search.WithSize(batchSize), - e.client.Search.WithBody(strings.NewReader(query)), + res, err := e.client.Bulk( + strings.NewReader(body.String()), + e.client.Bulk.WithContext(ctx), ) if err != nil { - result.Error = fmt.Sprintf("error in initial search: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result + return fmt.Errorf("bulk request error: %w", err) } defer res.Body.Close() if res.IsError() { - result.Error = fmt.Sprintf("elasticsearch error: %s", res.String()) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result + return fmt.Errorf("bulk request failed: %s", res.String()) } - var searchResult map[string]any - if err := json.NewDecoder(res.Body).Decode(&searchResult); err != nil { - result.Error = fmt.Sprintf("error decoding search result: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - - scrollID, ok := searchResult["_scroll_id"].(string) - if !ok { - result.Error = "no scroll_id found in search result" - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - - hits := searchResult["hits"].(map[string]any)["hits"].([]any) - firstDocument := true - documentCount := 0 - - for _, hit := range hits { - if !firstDocument { - if _, err := writer.Write([]byte(",\n")); err != nil { - result.Error = fmt.Sprintf("error writing separator: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - } - firstDocument = false - - source := hit.(map[string]any)["_source"] - if err := e.writeDocument(writer, source); err != nil { - result.Error = fmt.Sprintf("error writing document: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - documentCount++ - } - - for { - select { - case <-ctx.Done(): - result.Error = "context cancelled" - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - default: - } - - scrollRes, err := e.client.Scroll( - e.client.Scroll.WithScrollID(scrollID), - e.client.Scroll.WithScroll(1000), - e.client.Scroll.WithContext(ctx), - ) - if err != nil { - result.Error = fmt.Sprintf("error in scroll request: %v", err) - break - } - defer scrollRes.Body.Close() - - if scrollRes.IsError() { - result.Error = fmt.Sprintf("elasticsearch scroll error: %s", scrollRes.String()) - break - } - - var scrollResult map[string]any - if err := json.NewDecoder(scrollRes.Body).Decode(&scrollResult); err != nil { - result.Error = fmt.Sprintf("error decoding scroll result: %v", err) - break - } - - hits := scrollResult["hits"].(map[string]any)["hits"].([]any) - if len(hits) == 0 { - break - } - - scrollID, _ = scrollResult["_scroll_id"].(string) - - for _, hit := range hits { - if _, err := writer.Write([]byte(",\n")); err != nil { - result.Error = fmt.Sprintf("error writing separator: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - - source := hit.(map[string]any)["_source"] - if err := e.writeDocument(writer, source); err != nil { - result.Error = fmt.Sprintf("error writing document: %v", err) - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - return result - } - documentCount++ - } - } - - if _, err := writer.Write([]byte("\n ]")); err != nil { - if result.Error == "" { - result.Error = fmt.Sprintf("error writing index footer: %v", err) - } - } - - result.DocumentCount = documentCount - result.EndTime = time.Now() - result.Duration = time.Since(startTime).String() - - slog.Info("Index export completed", - "index", index, - "documents", documentCount, - "duration", result.Duration, - ) - - return result + slog.Debug("Batch successfully exported to Elasticsearch", "count", len(entries)) + return nil } -func (e *ElasticsearchExporter) writeDocument(writer io.Writer, document any) error { - jsonBytes, err := json.MarshalIndent(document, " ", " ") +func (e *ElasticsearchExporter) HealthCheck(ctx context.Context) error { + timeout := time.Duration(e.config.Timeout) * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + res, err := e.client.Info(e.client.Info.WithContext(ctx)) if err != nil { - return fmt.Errorf("error marshalling document: %w", err) + return fmt.Errorf("health check failed: %w", err) } + defer res.Body.Close() - if _, err := writer.Write([]byte(" ")); err != nil { - return err - } - - if _, err := writer.Write(jsonBytes); err != nil { - return err + if res.IsError() { + return fmt.Errorf("health check failed: %s", res.String()) } return nil diff --git a/elasticsearch_exporterV2.go b/elasticsearch_exporterV2.go deleted file mode 100644 index b0205e5..0000000 --- a/elasticsearch_exporterV2.go +++ /dev/null @@ -1,91 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "strings" - "time" - "tixel_watch/models" - - "github.com/elastic/go-elasticsearch/v8" -) - -type ElasticsearchExporterV2 struct { - client *elasticsearch.Client - config ElasticsearchConfig -} - -func NewElasticsearchExporterV2(config ElasticsearchConfig) (*ElasticsearchExporterV2, error) { - client, err := NewElasticsearchClient(config) - if err != nil { - return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err) - } - - return &ElasticsearchExporterV2{ - client: client, - config: config, - }, nil -} - -func (e *ElasticsearchExporterV2) Export(ctx context.Context, entries []models.LogMessage) error { - if len(entries) == 0 { - return nil - } - - var body strings.Builder - for _, entry := range entries { - indexName := e.config.Index - - indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName) - body.WriteString(indexLine) - body.WriteString("\n") - - data, err := json.Marshal(entry) - if err != nil { - slog.Error("error marshalling JSON", "error", err) - continue - } - body.WriteString(string(data)) - body.WriteString("\n") - } - - timeout := time.Duration(e.config.Timeout) * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - res, err := e.client.Bulk( - strings.NewReader(body.String()), - e.client.Bulk.WithContext(ctx), - ) - if err != nil { - return fmt.Errorf("bulk request error: %w", err) - } - defer res.Body.Close() - - if res.IsError() { - return fmt.Errorf("bulk request failed: %s", res.String()) - } - - slog.Debug("Batch successfully exported to Elasticsearch", "count", len(entries)) - return nil -} - -func (e *ElasticsearchExporterV2) HealthCheck(ctx context.Context) error { - timeout := time.Duration(e.config.Timeout) * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - res, err := e.client.Info(e.client.Info.WithContext(ctx)) - if err != nil { - return fmt.Errorf("health check failed: %w", err) - } - defer res.Body.Close() - - if res.IsError() { - return fmt.Errorf("health check failed: %s", res.String()) - } - - return nil -} diff --git a/export_manager.go b/export_manager.go index bf0ee72..a7fe501 100644 --- a/export_manager.go +++ b/export_manager.go @@ -8,7 +8,7 @@ import ( "strings" "sync" "time" - "tixel_watch/models" + "watch-tool/models" ) type ExportManager struct { diff --git a/exporter_interface.go b/exporter_interface.go index 4b9d455..5753f9e 100644 --- a/exporter_interface.go +++ b/exporter_interface.go @@ -2,7 +2,7 @@ package main import ( "context" - "tixel_watch/models" + "watch-tool/models" ) type ExporterInterface interface { diff --git a/file_monitor.go b/file_monitor.go index ab2c1de..85fc9a7 100644 --- a/file_monitor.go +++ b/file_monitor.go @@ -6,43 +6,52 @@ import ( "log/slog" "regexp" "strings" - "tixel_watch/models" - "tixel_watch/parser" + "watch-tool/models" + "watch-tool/parser" + "watch-tool/patterns" "github.com/hpcloud/tail" ) type FileMonitor struct { - config ToolConfig - parser parser.Parser + config ToolConfig + parser parser.Parser + hostname string } -func NewFileMonitor(config ToolConfig) *FileMonitor { +func NewFileMonitor(config ToolConfig, hostname string) *FileMonitor { var logParser parser.Parser if config.Format.Pattern != "" { - pattern, err := regexp.Compile(config.Format.Pattern) + compiledRegex, err := regexp.Compile(config.Format.Pattern) if err != nil { - slog.Error("invalid regex pattern", "tool", config.Name, "error", err) - logParser = &parser.DefaultParser{} + slog.Error("Invalid regex pattern in tool config", "tool", config.Name, "error", err) + logParser = parser.NewGenericParser(config.Name, hostname) } else { - logParser = &parser.RegexLogParser{ - Pattern: pattern, - Fields: config.Format.Fields, - Toolname: config.Name, + gp := parser.NewGenericParser(config.Name, hostname) + + customExtractor := patterns.CompiledExtractor{ + Name: "config_custom_pattern", + Pattern: compiledRegex, + Fields: config.Format.Fields, } + + gp.Extractors = append(gp.Extractors, customExtractor) + logParser = gp } } else { var err error - logParser, err = parser.New(config.Name, "custom") + logParser, err = parser.New(config.Name, "custom", hostname) if err != nil { - slog.Error("cannot get tool specific parser", "error", err) + slog.Error("Cannot get tool specific parser from factory", "error", err) + logParser = parser.NewGenericParser(config.Name, hostname) } } return &FileMonitor{ - config: config, - parser: logParser, + config: config, + parser: logParser, + hostname: hostname, } } @@ -72,7 +81,7 @@ func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage) } if line.Err != nil { - slog.Error("error reading log file", "tool", fm.config.Name, "error", line.Err) + slog.Error("Error reading log file", "tool", fm.config.Name, "error", line.Err) continue } @@ -82,7 +91,11 @@ func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage) entry, err := fm.parser.Parse(line.Text) if err != nil { - slog.Error("error parsing log line", "error", err) + slog.Error("Error parsing log line", "tool", fm.config.Name, "error", err) + } else { + if entry.Tool == "" { + entry.Tool = fm.config.Name + } } select { diff --git a/go.mod b/go.mod index fdc6247..15dd72a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module tixel_watch +module watch-tool go 1.24.1 @@ -8,6 +8,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.20.1 golang.org/x/sys v0.34.0 + gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.39.0 ) @@ -42,7 +43,6 @@ require ( golang.org/x/text v0.21.0 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.66.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/helpers/helpers.go b/helpers/helpers.go index e16183c..2a35776 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -3,11 +3,10 @@ package helpers import ( "fmt" "log/slog" - "os" "regexp" "strings" "time" - "tixel_watch/models" + "watch-tool/models" ) var ( @@ -76,11 +75,3 @@ func ParseSyslogTimeToRFC3339(syslogTime string) (time.Time, error) { t = t.AddDate(now.Year(), 0, 0) return t, nil } - -func GetHostname() (string, error) { - hostname, err := os.Hostname() - if err != nil { - hostname = "unknown" - } - return hostname, nil -} diff --git a/helpers/utils.go b/helpers/utils.go new file mode 100644 index 0000000..c8933bd --- /dev/null +++ b/helpers/utils.go @@ -0,0 +1,47 @@ +package helpers + +import ( + "context" + "fmt" + "log/slog" + "runtime/debug" +) + +type AppError struct { + Op string + Err error + Context string +} + +func (e *AppError) Error() string { + if e.Context != "" { + return fmt.Sprintf("%s: %v (%s)", e.Op, e.Err, e.Context) + } + return fmt.Sprintf("%s: %v", e.Op, e.Err) +} + +func (e *AppError) Unwrap() error { + return e.Err +} + +func NewAppError(op string, err error, ctx string) error { + return &AppError{Op: op, Err: err, Context: ctx} +} + +func SafeGo(ctx context.Context, name string, fn func()) { + go func() { + defer func() { + if r := recover(); r != nil { + stack := string(debug.Stack()) + slog.Error("CRITICAL: Panic recovered in goroutine", + "goroutine", name, + "panic", r, + "stack", stack, + ) + // Optional: Hier könnte man Metriken inkrementieren (siehe Observability) + } + }() + + fn() + }() +} diff --git a/local_storage.go b/local_storage.go index f4b6265..bafdb80 100644 --- a/local_storage.go +++ b/local_storage.go @@ -4,28 +4,277 @@ import ( "context" "database/sql" "encoding/json" - "tixel_watch/models" + "fmt" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + "watch-tool/models" _ "modernc.org/sqlite" ) -type StorageService struct { - db *sql.DB +type SQLiteStorage struct { + db *sql.DB + dbPath string + rotationCfg StorageRotationConfig + rotationStop chan struct{} + rotationWg sync.WaitGroup + mu sync.RWMutex } -func NewStorageService(dbPath string) (*StorageService, error) { - db, err := sql.Open("sqlite", dbPath) +func DefaultRotationConfig() StorageRotationConfig { + return StorageRotationConfig{ + MaxSizeBytes: 100 * 1024 * 1024, // 100MB + MaxAgeHours: 48 * time.Hour, // 48 hours + MaxFiles: 3, // 3 old Files + CheckIntervalMinutes: 5 * time.Minute, // check every 5 minutes + ArchiveDir: "", // same directory + } +} + +func NewSQLiteStorage(dbPath string) (*SQLiteStorage, error) { + return NewSQLiteStorageWithRotation(dbPath, StorageRotationConfig{}) +} + +func NewSQLiteStorageWithRotation(dbPath string, rotationCfg StorageRotationConfig) (*SQLiteStorage, error) { + if rotationCfg.CheckIntervalMinutes == 0 { + rotationCfg = DefaultRotationConfig() + } + dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", dbPath) + + db, err := sql.Open("sqlite", dsn) + + if err != nil { + return nil, fmt.Errorf("failed to open SQLite database: %w", err) + } + + if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil { + return nil, fmt.Errorf("failed to enable WAL mode: %w", err) + } + + if err := createTables(db); err != nil { + return nil, fmt.Errorf("failed to create tables: %w", err) + } + + storage := &SQLiteStorage{ + db: db, + dbPath: dbPath, + rotationCfg: rotationCfg, + rotationStop: make(chan struct{}), + } + + if rotationCfg.MaxSizeBytes > 0 || rotationCfg.MaxAgeHours > 0 { + storage.rotationWg.Add(1) + go storage.rotationWorker() + slog.Info("Log rotation enabled", + "maxSize", rotationCfg.MaxSizeBytes, + "maxAge", rotationCfg.MaxAgeHours, + "maxFiles", rotationCfg.MaxFiles) + } + + return storage, nil +} + +func (s *SQLiteStorage) rotationWorker() { + defer s.rotationWg.Done() + ticker := time.NewTicker(s.rotationCfg.CheckIntervalMinutes) + defer ticker.Stop() + + for { + select { + case <-s.rotationStop: + return + case <-ticker.C: + if err := s.checkAndRotate(); err != nil { + slog.Error("Error during log rotation check", "error", err) + } + } + } +} + +func (s *SQLiteStorage) checkAndRotate() error { + s.mu.Lock() + defer s.mu.Unlock() + + needsRotation, reason, err := s.needsRotation() + if err != nil { + return fmt.Errorf("error checking rotation needs: %w", err) + } + + if needsRotation { + slog.Info("Starting log rotation", "reason", reason) + if err := s.rotateDatabase(); err != nil { + return fmt.Errorf("error rotating database: %w", err) + } + slog.Info("Log rotation completed successfully") + } + + return nil +} + +func (s *SQLiteStorage) needsRotation() (bool, string, error) { + if s.rotationCfg.MaxSizeBytes > 0 { + fileInfo, err := os.Stat(s.dbPath) + if err != nil { + return false, "", err + } + if fileInfo.Size() >= s.rotationCfg.MaxSizeBytes { + return true, fmt.Sprintf("file size %d >= max size %d", fileInfo.Size(), s.rotationCfg.MaxSizeBytes), nil + } + } + + if s.rotationCfg.MaxAgeHours > 0 { + fileInfo, err := os.Stat(s.dbPath) + if err != nil { + return false, "", err + } + age := time.Since(fileInfo.ModTime()) + if age >= s.rotationCfg.MaxAgeHours { + return true, fmt.Sprintf("file age %v >= max age %v", age, s.rotationCfg.MaxAgeHours), nil + } + } + + return false, "", nil +} + +func (s *SQLiteStorage) rotateDatabase() error { + if err := s.db.Close(); err != nil { + return fmt.Errorf("error closing database: %w", err) + } + + archivePath := s.generateArchivePath() + + if err := os.Rename(s.dbPath, archivePath); err != nil { + return fmt.Errorf("error moving database to archive: %w", err) + } + + db, err := sql.Open("sqlite", s.dbPath) + if err != nil { + return fmt.Errorf("error opening new database: %w", err) + } + + if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil { + return fmt.Errorf("failed to enable WAL mode on new database: %w", err) + } + + if err := createTables(db); err != nil { + return fmt.Errorf("failed to create tables in new database: %w", err) + } + + s.db = db + + if err := s.cleanupOldArchives(); err != nil { + slog.Warn("Error cleaning up old archives", "error", err) + } + + return nil +} + +func (s *SQLiteStorage) generateArchivePath() string { + dir := filepath.Dir(s.dbPath) + if s.rotationCfg.ArchiveDir != "" { + dir = s.rotationCfg.ArchiveDir + os.MkdirAll(dir, 0755) + } + + base := filepath.Base(s.dbPath) + ext := filepath.Ext(base) + name := strings.TrimSuffix(base, ext) + + timestamp := time.Now().Format("2006-01-02_15-04-05") + archiveName := fmt.Sprintf("%s.%s%s", name, timestamp, ext) + + return filepath.Join(dir, archiveName) +} + +func (s *SQLiteStorage) cleanupOldArchives() error { + if s.rotationCfg.MaxFiles <= 0 { + return nil + } + + dir := filepath.Dir(s.dbPath) + if s.rotationCfg.ArchiveDir != "" { + dir = s.rotationCfg.ArchiveDir + } + + base := filepath.Base(s.dbPath) + ext := filepath.Ext(base) + name := strings.TrimSuffix(base, ext) + pattern := fmt.Sprintf("%s.*%s", name, ext) + + files, err := filepath.Glob(filepath.Join(dir, pattern)) + if err != nil { + return err + } + + var archives []string + for _, file := range files { + if file != s.dbPath { + archives = append(archives, file) + } + } + + sort.Slice(archives, func(i, j int) bool { + infoI, _ := os.Stat(archives[i]) + infoJ, _ := os.Stat(archives[j]) + return infoI.ModTime().After(infoJ.ModTime()) + }) + + if len(archives) > s.rotationCfg.MaxFiles { + for _, file := range archives[s.rotationCfg.MaxFiles:] { + if err := os.Remove(file); err != nil { + slog.Warn("Error removing old archive", "file", file, "error", err) + } else { + slog.Info("Removed old archive", "file", file) + } + } + } + + return nil +} + +func (s *SQLiteStorage) ForceRotate() error { + s.mu.Lock() + defer s.mu.Unlock() + + slog.Info("Forcing log rotation") + return s.rotateDatabase() +} + +func (s *SQLiteStorage) GetRotationInfo() (map[string]any, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + fileInfo, err := os.Stat(s.dbPath) if err != nil { return nil, err } + info := map[string]any{ + "currentSize": fileInfo.Size(), + "maxSize": s.rotationCfg.MaxSizeBytes, + "currentAge": time.Since(fileInfo.ModTime()).String(), + "maxAge": s.rotationCfg.MaxAgeHours.String(), + "maxFiles": s.rotationCfg.MaxFiles, + "checkInterval": s.rotationCfg.CheckIntervalMinutes.String(), + "archiveDir": s.rotationCfg.ArchiveDir, + } + + return info, nil +} + +func createTables(db *sql.DB) error { createTableStmt := ` CREATE TABLE IF NOT EXISTS log_entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, service TEXT, - timestamp DATETIME, - type TEXT, - host TEXT, + timestamp DATETIME NOT NULL, + type TEXT NOT NULL, + host TEXT NOT NULL, tool TEXT, log_level TEXT, log_message TEXT, @@ -39,143 +288,317 @@ func NewStorageService(dbPath string) (*StorageService, error) { fields TEXT, service_information TEXT, system_metrics TEXT, - tool_information TEXT - ); - ` - _, err = db.ExecContext(context.Background(), createTableStmt) - if err != nil { - return nil, err + tool_information TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + exported_at DATETIME + );` + + if _, err := db.Exec(createTableStmt); err != nil { + return err } - return &StorageService{db: db}, nil + indexes := []string{ + "CREATE INDEX IF NOT EXISTS idx_timestamp ON log_entries(timestamp);", + "CREATE INDEX IF NOT EXISTS idx_service ON log_entries(service);", + "CREATE INDEX IF NOT EXISTS idx_type ON log_entries(type);", + "CREATE INDEX IF NOT EXISTS idx_tool ON log_entries(tool);", + "CREATE INDEX IF NOT EXISTS idx_log_level ON log_entries(log_level);", + "CREATE INDEX IF NOT EXISTS idx_exported ON log_entries(exported_at);", + "CREATE INDEX IF NOT EXISTS idx_composite ON log_entries(timestamp, type, service);", + } + + for _, index := range indexes { + if _, err := db.Exec(index); err != nil { + return fmt.Errorf("failed to create index: %w", err) + } + } + + return nil } -func (s *StorageService) Close() error { +func (s *SQLiteStorage) Store(ctx context.Context, entry *models.LogMessage) error { + return s.StoreBatch(ctx, []models.LogMessage{*entry}) +} + +func (s *SQLiteStorage) StoreBatch(ctx context.Context, entries []models.LogMessage) error { + if len(entries) == 0 { + return nil + } + + s.mu.RLock() + defer s.mu.RUnlock() + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.PrepareContext(ctx, ` + INSERT INTO log_entries + (service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, + unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, entry := range entries { + fieldsJSON, _ := json.Marshal(entry.Fields) + serviceInfoJSON, _ := json.Marshal(entry.ServiceInformation) + systemMetricsJSON, _ := json.Marshal(entry.SystemMetrics) + toolInfoJSON, _ := json.Marshal(entry.ToolInformation) + + _, err := stmt.ExecContext(ctx, + entry.Service, + entry.Timestamp, + entry.Type, + entry.Host, + entry.Tool, + entry.LogLevel, + entry.LogMessage, + entry.Raw, + entry.Priority, + entry.PriorityName, + entry.Unit, + entry.PID, + entry.BootID, + entry.MachineID, + string(fieldsJSON), + string(serviceInfoJSON), + string(systemMetricsJSON), + string(toolInfoJSON), + ) + if err != nil { + return fmt.Errorf("failed to insert entry: %w", err) + } + } + + return tx.Commit() +} + +func (s *SQLiteStorage) Query(ctx context.Context, query StorageQuery) ([]models.LogMessage, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + sqlQuery := "SELECT service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information FROM log_entries WHERE 1=1" + args := []any{} + argCount := 0 + + if !query.StartTime.IsZero() { + argCount++ + sqlQuery += fmt.Sprintf(" AND timestamp >= ?%d", argCount) + args = append(args, query.StartTime) + } + if !query.EndTime.IsZero() { + argCount++ + sqlQuery += fmt.Sprintf(" AND timestamp <= ?%d", argCount) + args = append(args, query.EndTime) + } + if query.Service != "" { + argCount++ + sqlQuery += fmt.Sprintf(" AND service = ?%d", argCount) + args = append(args, query.Service) + } + if query.Tool != "" { + argCount++ + sqlQuery += fmt.Sprintf(" AND tool = ?%d", argCount) + args = append(args, query.Tool) + } + if query.LogLevel != "" { + argCount++ + sqlQuery += fmt.Sprintf(" AND log_level = ?%d", argCount) + args = append(args, query.LogLevel) + } + if query.Type != "" { + argCount++ + sqlQuery += fmt.Sprintf(" AND type = ?%d", argCount) + args = append(args, query.Type) + } + + if query.OrderBy != "" { + direction := "ASC" + if query.OrderDesc { + direction = "DESC" + } + sqlQuery += fmt.Sprintf(" ORDER BY %s %s", query.OrderBy, direction) + } else { + sqlQuery += " ORDER BY timestamp DESC" + } + + if query.Limit > 0 { + sqlQuery += fmt.Sprintf(" LIMIT %d", query.Limit) + if query.Offset > 0 { + sqlQuery += fmt.Sprintf(" OFFSET %d", query.Offset) + } + } + + rows, err := s.db.QueryContext(ctx, sqlQuery, args...) + if err != nil { + return nil, fmt.Errorf("failed to execute query: %w", err) + } + defer rows.Close() + + var entries []models.LogMessage + for rows.Next() { + var entry models.LogMessage + var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string + + err := rows.Scan( + &entry.Service, + &entry.Timestamp, + &entry.Type, + &entry.Host, + &entry.Tool, + &entry.LogLevel, + &entry.LogMessage, + &entry.Raw, + &entry.Priority, + &entry.PriorityName, + &entry.Unit, + &entry.PID, + &entry.BootID, + &entry.MachineID, + &fieldsJSON, + &serviceInfoJSON, + &systemMetricsJSON, + &toolInfoJSON, + ) + if err != nil { + return nil, fmt.Errorf("failed to scan row: %w", err) + } + + if fieldsJSON != "" && fieldsJSON != "null" { + json.Unmarshal([]byte(fieldsJSON), &entry.Fields) + } + if serviceInfoJSON != "" && serviceInfoJSON != "null" { + json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation) + } + if systemMetricsJSON != "" && systemMetricsJSON != "null" { + json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics) + } + if toolInfoJSON != "" && toolInfoJSON != "null" { + json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation) + } + + entries = append(entries, entry) + } + + return entries, rows.Err() +} + +func (s *SQLiteStorage) MarkAsExported(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + + s.mu.RLock() + defer s.mu.RUnlock() + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + + placeholders := strings.Repeat("?,", len(ids)) + placeholders = placeholders[:len(placeholders)-1] + + sqlQuery := fmt.Sprintf("UPDATE log_entries SET exported_at = CURRENT_TIMESTAMP WHERE id IN (%s)", placeholders) + + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + + _, err = tx.ExecContext(ctx, sqlQuery, args...) + if err != nil { + tx.Rollback() + return err + } + + return tx.Commit() +} + +func (s *SQLiteStorage) GetUnexportedEntries(ctx context.Context, limit int) ([]models.LogMessage, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + sqlQuery := `SELECT id, service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, + unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information + FROM log_entries WHERE exported_at IS NULL ORDER BY timestamp ASC` + + if limit > 0 { + sqlQuery += fmt.Sprintf(" LIMIT %d", limit) + } + + rows, err := s.db.QueryContext(ctx, sqlQuery) + if err != nil { + return nil, fmt.Errorf("failed to execute query: %w", err) + } + defer rows.Close() + + var entries []models.LogMessage + for rows.Next() { + var entry models.LogMessage + var id int64 + var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string + + err := rows.Scan( + &id, + &entry.Service, + &entry.Timestamp, + &entry.Type, + &entry.Host, + &entry.Tool, + &entry.LogLevel, + &entry.LogMessage, + &entry.Raw, + &entry.Priority, + &entry.PriorityName, + &entry.Unit, + &entry.PID, + &entry.BootID, + &entry.MachineID, + &fieldsJSON, + &serviceInfoJSON, + &systemMetricsJSON, + &toolInfoJSON, + ) + if err != nil { + return nil, fmt.Errorf("failed to scan row: %w", err) + } + + if entry.Fields == nil { + entry.Fields = make(map[string]any) + } + entry.Fields["_internal_id"] = id + + if fieldsJSON != "" && fieldsJSON != "null" { + json.Unmarshal([]byte(fieldsJSON), &entry.Fields) + } + if serviceInfoJSON != "" && serviceInfoJSON != "null" { + json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation) + } + if systemMetricsJSON != "" && systemMetricsJSON != "null" { + json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics) + } + if toolInfoJSON != "" && toolInfoJSON != "null" { + json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation) + } + + entries = append(entries, entry) + } + + return entries, rows.Err() +} + +func (s *SQLiteStorage) Close() error { + close(s.rotationStop) + s.rotationWg.Wait() + + s.mu.Lock() + defer s.mu.Unlock() + return s.db.Close() } - -func (s *StorageService) SaveLogEntry(ctx context.Context, entry *models.LogMessage) error { - fieldsJSON := "" - if entry.Fields != nil { - b, err := json.Marshal(entry.Fields) - if err != nil { - return err - } - fieldsJSON = string(b) - } - - serviceInfoJSON := "" - if entry.ServiceInformation != nil { - b, err := json.Marshal(entry.ServiceInformation) - if err != nil { - return err - } - serviceInfoJSON = string(b) - } - - systemMetricsJSON := "" - if entry.SystemMetrics != nil { - b, err := json.Marshal(entry.SystemMetrics) - if err != nil { - return err - } - systemMetricsJSON = string(b) - } - - toolInfoJSON := "" - if entry.ToolInformation != nil { - b, err := json.Marshal(entry.ToolInformation) - if err != nil { - return err - } - toolInfoJSON = string(b) - } - - stmt := ` - INSERT INTO log_entries - (service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ` - _, err := s.db.ExecContext(ctx, stmt, - entry.Service, - entry.Timestamp, - entry.Type, - entry.Host, - entry.Tool, - entry.LogLevel, - entry.LogMessage, - entry.Raw, - entry.Priority, - entry.PriorityName, - entry.Unit, - entry.PID, - entry.BootID, - entry.MachineID, - fieldsJSON, - serviceInfoJSON, - systemMetricsJSON, - toolInfoJSON, - ) - return err -} - -func (s *StorageService) LoadLogEntry(ctx context.Context, id int64) (*models.LogMessage, error) { - row := s.db.QueryRowContext(ctx, "SELECT service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information FROM log_entries WHERE id = ?", id) - - var entry models.LogMessage - var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string - - err := row.Scan( - &entry.Service, - &entry.Timestamp, - &entry.Type, - &entry.Host, - &entry.Tool, - &entry.LogLevel, - &entry.LogMessage, - &entry.Raw, - &entry.Priority, - &entry.PriorityName, - &entry.Unit, - &entry.PID, - &entry.BootID, - &entry.MachineID, - &fieldsJSON, - &serviceInfoJSON, - &systemMetricsJSON, - &toolInfoJSON, - ) - if err != nil { - return nil, err - } - - if fieldsJSON != "" { - var fields map[string]any - if err = json.Unmarshal([]byte(fieldsJSON), &fields); err == nil { - entry.Fields = fields - } - } - - if serviceInfoJSON != "" { - var si any - if err = json.Unmarshal([]byte(serviceInfoJSON), &si); err == nil { - entry.ServiceInformation = si - } - } - - if systemMetricsJSON != "" { - var sm any - if err = json.Unmarshal([]byte(systemMetricsJSON), &sm); err == nil { - entry.SystemMetrics = sm - } - } - - if toolInfoJSON != "" { - var ti any - if err = json.Unmarshal([]byte(toolInfoJSON), &ti); err == nil { - entry.ToolInformation = ti - } - } - - return &entry, nil -} diff --git a/local_storageV2.go b/local_storageV2.go deleted file mode 100644 index f6c7dea..0000000 --- a/local_storageV2.go +++ /dev/null @@ -1,602 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "log/slog" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - "tixel_watch/models" - - _ "modernc.org/sqlite" -) - -type SQLiteStorage struct { - db *sql.DB - dbPath string - rotationCfg StorageRotationConfig - rotationStop chan struct{} - rotationWg sync.WaitGroup - mu sync.RWMutex -} - -func DefaultRotationConfig() StorageRotationConfig { - return StorageRotationConfig{ - MaxSizeBytes: 100 * 1024 * 1024, // 100MB - MaxAgeHours: 48 * time.Hour, // 48 hours - MaxFiles: 3, // 3 old Files - CheckIntervalMinutes: 5 * time.Minute, // check every 5 minutes - ArchiveDir: "", // same directory - } -} - -func NewSQLiteStorage(dbPath string) (*SQLiteStorage, error) { - return NewSQLiteStorageWithRotation(dbPath, StorageRotationConfig{}) -} - -func NewSQLiteStorageWithRotation(dbPath string, rotationCfg StorageRotationConfig) (*SQLiteStorage, error) { - if rotationCfg.CheckIntervalMinutes == 0 { - rotationCfg = DefaultRotationConfig() - } - - db, err := sql.Open("sqlite", dbPath) - if err != nil { - return nil, fmt.Errorf("failed to open SQLite database: %w", err) - } - - if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil { - return nil, fmt.Errorf("failed to enable WAL mode: %w", err) - } - - if err := createTables(db); err != nil { - return nil, fmt.Errorf("failed to create tables: %w", err) - } - - storage := &SQLiteStorage{ - db: db, - dbPath: dbPath, - rotationCfg: rotationCfg, - rotationStop: make(chan struct{}), - } - - if rotationCfg.MaxSizeBytes > 0 || rotationCfg.MaxAgeHours > 0 { - storage.rotationWg.Add(1) - go storage.rotationWorker() - slog.Info("Log rotation enabled", - "maxSize", rotationCfg.MaxSizeBytes, - "maxAge", rotationCfg.MaxAgeHours, - "maxFiles", rotationCfg.MaxFiles) - } - - return storage, nil -} - -func (s *SQLiteStorage) rotationWorker() { - defer s.rotationWg.Done() - ticker := time.NewTicker(s.rotationCfg.CheckIntervalMinutes) - defer ticker.Stop() - - for { - select { - case <-s.rotationStop: - return - case <-ticker.C: - if err := s.checkAndRotate(); err != nil { - slog.Error("Error during log rotation check", "error", err) - } - } - } -} - -func (s *SQLiteStorage) checkAndRotate() error { - s.mu.Lock() - defer s.mu.Unlock() - - needsRotation, reason, err := s.needsRotation() - if err != nil { - return fmt.Errorf("error checking rotation needs: %w", err) - } - - if needsRotation { - slog.Info("Starting log rotation", "reason", reason) - if err := s.rotateDatabase(); err != nil { - return fmt.Errorf("error rotating database: %w", err) - } - slog.Info("Log rotation completed successfully") - } - - return nil -} - -func (s *SQLiteStorage) needsRotation() (bool, string, error) { - if s.rotationCfg.MaxSizeBytes > 0 { - fileInfo, err := os.Stat(s.dbPath) - if err != nil { - return false, "", err - } - if fileInfo.Size() >= s.rotationCfg.MaxSizeBytes { - return true, fmt.Sprintf("file size %d >= max size %d", fileInfo.Size(), s.rotationCfg.MaxSizeBytes), nil - } - } - - if s.rotationCfg.MaxAgeHours > 0 { - fileInfo, err := os.Stat(s.dbPath) - if err != nil { - return false, "", err - } - age := time.Since(fileInfo.ModTime()) - if age >= s.rotationCfg.MaxAgeHours { - return true, fmt.Sprintf("file age %v >= max age %v", age, s.rotationCfg.MaxAgeHours), nil - } - } - - return false, "", nil -} - -func (s *SQLiteStorage) rotateDatabase() error { - if err := s.db.Close(); err != nil { - return fmt.Errorf("error closing database: %w", err) - } - - archivePath := s.generateArchivePath() - - if err := os.Rename(s.dbPath, archivePath); err != nil { - return fmt.Errorf("error moving database to archive: %w", err) - } - - db, err := sql.Open("sqlite", s.dbPath) - if err != nil { - return fmt.Errorf("error opening new database: %w", err) - } - - if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil { - return fmt.Errorf("failed to enable WAL mode on new database: %w", err) - } - - if err := createTables(db); err != nil { - return fmt.Errorf("failed to create tables in new database: %w", err) - } - - s.db = db - - if err := s.cleanupOldArchives(); err != nil { - slog.Warn("Error cleaning up old archives", "error", err) - } - - return nil -} - -func (s *SQLiteStorage) generateArchivePath() string { - dir := filepath.Dir(s.dbPath) - if s.rotationCfg.ArchiveDir != "" { - dir = s.rotationCfg.ArchiveDir - os.MkdirAll(dir, 0755) - } - - base := filepath.Base(s.dbPath) - ext := filepath.Ext(base) - name := strings.TrimSuffix(base, ext) - - timestamp := time.Now().Format("2006-01-02_15-04-05") - archiveName := fmt.Sprintf("%s.%s%s", name, timestamp, ext) - - return filepath.Join(dir, archiveName) -} - -func (s *SQLiteStorage) cleanupOldArchives() error { - if s.rotationCfg.MaxFiles <= 0 { - return nil - } - - dir := filepath.Dir(s.dbPath) - if s.rotationCfg.ArchiveDir != "" { - dir = s.rotationCfg.ArchiveDir - } - - base := filepath.Base(s.dbPath) - ext := filepath.Ext(base) - name := strings.TrimSuffix(base, ext) - pattern := fmt.Sprintf("%s.*%s", name, ext) - - files, err := filepath.Glob(filepath.Join(dir, pattern)) - if err != nil { - return err - } - - var archives []string - for _, file := range files { - if file != s.dbPath { - archives = append(archives, file) - } - } - - sort.Slice(archives, func(i, j int) bool { - infoI, _ := os.Stat(archives[i]) - infoJ, _ := os.Stat(archives[j]) - return infoI.ModTime().After(infoJ.ModTime()) - }) - - if len(archives) > s.rotationCfg.MaxFiles { - for _, file := range archives[s.rotationCfg.MaxFiles:] { - if err := os.Remove(file); err != nil { - slog.Warn("Error removing old archive", "file", file, "error", err) - } else { - slog.Info("Removed old archive", "file", file) - } - } - } - - return nil -} - -func (s *SQLiteStorage) ForceRotate() error { - s.mu.Lock() - defer s.mu.Unlock() - - slog.Info("Forcing log rotation") - return s.rotateDatabase() -} - -func (s *SQLiteStorage) GetRotationInfo() (map[string]any, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - fileInfo, err := os.Stat(s.dbPath) - if err != nil { - return nil, err - } - - info := map[string]any{ - "currentSize": fileInfo.Size(), - "maxSize": s.rotationCfg.MaxSizeBytes, - "currentAge": time.Since(fileInfo.ModTime()).String(), - "maxAge": s.rotationCfg.MaxAgeHours.String(), - "maxFiles": s.rotationCfg.MaxFiles, - "checkInterval": s.rotationCfg.CheckIntervalMinutes.String(), - "archiveDir": s.rotationCfg.ArchiveDir, - } - - return info, nil -} - -func createTables(db *sql.DB) error { - createTableStmt := ` - CREATE TABLE IF NOT EXISTS log_entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - service TEXT, - timestamp DATETIME NOT NULL, - type TEXT NOT NULL, - host TEXT NOT NULL, - tool TEXT, - log_level TEXT, - log_message TEXT, - raw TEXT, - priority TEXT, - priority_name TEXT, - unit TEXT, - pid INTEGER, - boot_id TEXT, - machine_id TEXT, - fields TEXT, - service_information TEXT, - system_metrics TEXT, - tool_information TEXT, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - exported_at DATETIME - );` - - if _, err := db.Exec(createTableStmt); err != nil { - return err - } - - indexes := []string{ - "CREATE INDEX IF NOT EXISTS idx_timestamp ON log_entries(timestamp);", - "CREATE INDEX IF NOT EXISTS idx_service ON log_entries(service);", - "CREATE INDEX IF NOT EXISTS idx_type ON log_entries(type);", - "CREATE INDEX IF NOT EXISTS idx_tool ON log_entries(tool);", - "CREATE INDEX IF NOT EXISTS idx_log_level ON log_entries(log_level);", - "CREATE INDEX IF NOT EXISTS idx_exported ON log_entries(exported_at);", - "CREATE INDEX IF NOT EXISTS idx_composite ON log_entries(timestamp, type, service);", - } - - for _, index := range indexes { - if _, err := db.Exec(index); err != nil { - return fmt.Errorf("failed to create index: %w", err) - } - } - - return nil -} - -func (s *SQLiteStorage) Store(ctx context.Context, entry *models.LogMessage) error { - return s.StoreBatch(ctx, []models.LogMessage{*entry}) -} - -func (s *SQLiteStorage) StoreBatch(ctx context.Context, entries []models.LogMessage) error { - if len(entries) == 0 { - return nil - } - - s.mu.RLock() - defer s.mu.RUnlock() - - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer tx.Rollback() - - stmt, err := tx.PrepareContext(ctx, ` - INSERT INTO log_entries - (service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, - unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer stmt.Close() - - for _, entry := range entries { - fieldsJSON, _ := json.Marshal(entry.Fields) - serviceInfoJSON, _ := json.Marshal(entry.ServiceInformation) - systemMetricsJSON, _ := json.Marshal(entry.SystemMetrics) - toolInfoJSON, _ := json.Marshal(entry.ToolInformation) - - _, err := stmt.ExecContext(ctx, - entry.Service, - entry.Timestamp, - entry.Type, - entry.Host, - entry.Tool, - entry.LogLevel, - entry.LogMessage, - entry.Raw, - entry.Priority, - entry.PriorityName, - entry.Unit, - entry.PID, - entry.BootID, - entry.MachineID, - string(fieldsJSON), - string(serviceInfoJSON), - string(systemMetricsJSON), - string(toolInfoJSON), - ) - if err != nil { - return fmt.Errorf("failed to insert entry: %w", err) - } - } - - return tx.Commit() -} - -func (s *SQLiteStorage) Query(ctx context.Context, query StorageQuery) ([]models.LogMessage, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - sqlQuery := "SELECT service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information FROM log_entries WHERE 1=1" - args := []any{} - argCount := 0 - - if !query.StartTime.IsZero() { - argCount++ - sqlQuery += fmt.Sprintf(" AND timestamp >= ?%d", argCount) - args = append(args, query.StartTime) - } - if !query.EndTime.IsZero() { - argCount++ - sqlQuery += fmt.Sprintf(" AND timestamp <= ?%d", argCount) - args = append(args, query.EndTime) - } - if query.Service != "" { - argCount++ - sqlQuery += fmt.Sprintf(" AND service = ?%d", argCount) - args = append(args, query.Service) - } - if query.Tool != "" { - argCount++ - sqlQuery += fmt.Sprintf(" AND tool = ?%d", argCount) - args = append(args, query.Tool) - } - if query.LogLevel != "" { - argCount++ - sqlQuery += fmt.Sprintf(" AND log_level = ?%d", argCount) - args = append(args, query.LogLevel) - } - if query.Type != "" { - argCount++ - sqlQuery += fmt.Sprintf(" AND type = ?%d", argCount) - args = append(args, query.Type) - } - - if query.OrderBy != "" { - direction := "ASC" - if query.OrderDesc { - direction = "DESC" - } - sqlQuery += fmt.Sprintf(" ORDER BY %s %s", query.OrderBy, direction) - } else { - sqlQuery += " ORDER BY timestamp DESC" - } - - if query.Limit > 0 { - sqlQuery += fmt.Sprintf(" LIMIT %d", query.Limit) - if query.Offset > 0 { - sqlQuery += fmt.Sprintf(" OFFSET %d", query.Offset) - } - } - - rows, err := s.db.QueryContext(ctx, sqlQuery, args...) - if err != nil { - return nil, fmt.Errorf("failed to execute query: %w", err) - } - defer rows.Close() - - var entries []models.LogMessage - for rows.Next() { - var entry models.LogMessage - var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string - - err := rows.Scan( - &entry.Service, - &entry.Timestamp, - &entry.Type, - &entry.Host, - &entry.Tool, - &entry.LogLevel, - &entry.LogMessage, - &entry.Raw, - &entry.Priority, - &entry.PriorityName, - &entry.Unit, - &entry.PID, - &entry.BootID, - &entry.MachineID, - &fieldsJSON, - &serviceInfoJSON, - &systemMetricsJSON, - &toolInfoJSON, - ) - if err != nil { - return nil, fmt.Errorf("failed to scan row: %w", err) - } - - if fieldsJSON != "" && fieldsJSON != "null" { - json.Unmarshal([]byte(fieldsJSON), &entry.Fields) - } - if serviceInfoJSON != "" && serviceInfoJSON != "null" { - json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation) - } - if systemMetricsJSON != "" && systemMetricsJSON != "null" { - json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics) - } - if toolInfoJSON != "" && toolInfoJSON != "null" { - json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation) - } - - entries = append(entries, entry) - } - - return entries, rows.Err() -} - -func (s *SQLiteStorage) MarkAsExported(ctx context.Context, ids []int64) error { - if len(ids) == 0 { - return nil - } - - s.mu.RLock() - defer s.mu.RUnlock() - - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - - placeholders := strings.Repeat("?,", len(ids)) - placeholders = placeholders[:len(placeholders)-1] - - sqlQuery := fmt.Sprintf("UPDATE log_entries SET exported_at = CURRENT_TIMESTAMP WHERE id IN (%s)", placeholders) - - args := make([]any, len(ids)) - for i, id := range ids { - args[i] = id - } - - _, err = tx.ExecContext(ctx, sqlQuery, args...) - if err != nil { - tx.Rollback() - return err - } - - return tx.Commit() -} - -func (s *SQLiteStorage) GetUnexportedEntries(ctx context.Context, limit int) ([]models.LogMessage, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - sqlQuery := `SELECT id, service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, - unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information - FROM log_entries WHERE exported_at IS NULL ORDER BY timestamp ASC` - - if limit > 0 { - sqlQuery += fmt.Sprintf(" LIMIT %d", limit) - } - - rows, err := s.db.QueryContext(ctx, sqlQuery) - if err != nil { - return nil, fmt.Errorf("failed to execute query: %w", err) - } - defer rows.Close() - - var entries []models.LogMessage - for rows.Next() { - var entry models.LogMessage - var id int64 - var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string - - err := rows.Scan( - &id, - &entry.Service, - &entry.Timestamp, - &entry.Type, - &entry.Host, - &entry.Tool, - &entry.LogLevel, - &entry.LogMessage, - &entry.Raw, - &entry.Priority, - &entry.PriorityName, - &entry.Unit, - &entry.PID, - &entry.BootID, - &entry.MachineID, - &fieldsJSON, - &serviceInfoJSON, - &systemMetricsJSON, - &toolInfoJSON, - ) - if err != nil { - return nil, fmt.Errorf("failed to scan row: %w", err) - } - - if entry.Fields == nil { - entry.Fields = make(map[string]any) - } - entry.Fields["_internal_id"] = id - - if fieldsJSON != "" && fieldsJSON != "null" { - json.Unmarshal([]byte(fieldsJSON), &entry.Fields) - } - if serviceInfoJSON != "" && serviceInfoJSON != "null" { - json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation) - } - if systemMetricsJSON != "" && systemMetricsJSON != "null" { - json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics) - } - if toolInfoJSON != "" && toolInfoJSON != "null" { - json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation) - } - - entries = append(entries, entry) - } - - return entries, rows.Err() -} - -func (s *SQLiteStorage) Close() error { - close(s.rotationStop) - s.rotationWg.Wait() - - s.mu.Lock() - defer s.mu.Unlock() - - return s.db.Close() -} diff --git a/log_processor.go b/log_processor.go index ab0fd25..cb11b84 100644 --- a/log_processor.go +++ b/log_processor.go @@ -4,7 +4,7 @@ import ( "context" "log/slog" "time" - "tixel_watch/models" + "watch-tool/models" ) type LogProcessor struct { diff --git a/main.go b/main.go index 9ae138a..5c67513 100644 --- a/main.go +++ b/main.go @@ -8,26 +8,36 @@ import ( "sync" "syscall" "time" - "tixel_watch/models" + "watch-tool/helpers" + "watch-tool/models" + "watch-tool/patterns" ) -var hostname string +var currentHostname string func init() { var err error - hostname, err = os.Hostname() + currentHostname, err = os.Hostname() if err != nil { - hostname = "unknown" + currentHostname = "unknown" + slog.Warn("Could not determine hostname, using fallback", "fallback", currentHostname) } } func main() { - cfg, err := LoadConfigV2() + cfg, err := LoadConfig() if err != nil { - slog.Error("error loading configuration", "error", err) + slog.Error("Startup failed: configuration error", "error", err) os.Exit(1) } - slog.Info("TIXEL System Monitor started") + + slog.Info("System Monitor started", "hostname", currentHostname) + + if err := patterns.GetInstance().Load(cfg.PatternsFile); err != nil { + slog.Error("Startup failed: could not load patterns", "file", cfg.PatternsFile, "error", err) + os.Exit(1) + } + slog.Info("Regex patterns loaded successfully", "file", cfg.PatternsFile) var storage StorageInterface if cfg.LocalStorage.Enable { @@ -46,7 +56,7 @@ func main() { } storage = sqliteStorage defer storage.Close() - slog.Info("SQLite storage with rotation initialized", "path", cfg.LocalStorage.DBPath) + slog.Info("SQLite storage initialized", "path", cfg.LocalStorage.DBPath) } else { slog.Error("Local storage is disabled, but it's required for the new architecture") os.Exit(1) @@ -65,7 +75,7 @@ func main() { exportManager = NewExportManager(storage, exportConfig) if cfg.Elasticsearch.Enabled { - esExporter, err := NewElasticsearchExporterV2(cfg.Elasticsearch) + esExporter, err := NewElasticsearchExporter(cfg.Elasticsearch) if err != nil { slog.Error("failed to create Elasticsearch exporter", "error", err) os.Exit(1) @@ -79,10 +89,6 @@ func main() { exportManager.RegisterExporter("elasticsearch", esExporter) slog.Info("Elasticsearch exporter registered") } - - // Add more exporters here in the future - // exportManager.RegisterExporter("checkmk", checkmkExporter) - // exportManager.RegisterExporter("grafana", grafanaExporter) } logChan := make(chan models.LogMessage, 1000) @@ -92,86 +98,92 @@ func main() { var wg sync.WaitGroup wg.Add(1) - go func() { + helpers.SafeGo(ctx, "LogProcessor", func() { defer wg.Done() processor := NewLogProcessor(storage) processor.Start(ctx, logChan) - }() + }) if exportManager != nil { wg.Add(1) - go func() { + helpers.SafeGo(ctx, "ExportManager", func() { defer wg.Done() exportManager.Start(ctx) - }() + }) } for _, service := range cfg.Services { if !service.Enabled { - slog.Info("Service deactivated, skipping...", "service", service.Name) + slog.Debug("Service deactivated, skipping...", "service", service.Name) continue } wg.Add(1) - go func(s ServiceConfig) { - defer wg.Done() - monitor := NewServiceMonitor(s) - if err := monitor.Start(ctx, logChan); err != nil { - slog.Error("error watching service", "service", s.Name, "error", err) - } - }(service) + srv := service - slog.Info("started watching Service-Log", "service", service.Name) + helpers.SafeGo(ctx, "ServiceMonitor-"+srv.Name, func() { + defer wg.Done() + monitor := NewServiceMonitor(srv, currentHostname) + + if err := monitor.Start(ctx, logChan); err != nil { + slog.Error("Error watching service", "service", srv.Name, "error", err) + } + }) + + slog.Info("Started watching Service-Log", "service", service.Name) } for _, tool := range cfg.Tools { if !tool.Enabled { - slog.Info("Tool is deactivated, skipping...", "tool", tool.Name) + slog.Debug("Tool is deactivated, skipping...", "tool", tool.Name) continue } wg.Add(1) - go func(t ToolConfig) { - defer wg.Done() - monitor := NewFileMonitor(t) - if err := monitor.Start(ctx, logChan); err != nil { - slog.Error("error watching", "tool", t.Name, "error", err) - } - }(tool) + t := tool - slog.Info("started watching logs", "tool", tool.Name, "file", tool.LogFile) + helpers.SafeGo(ctx, "FileMonitor-"+t.Name, func() { + defer wg.Done() + + monitor := NewFileMonitor(t, currentHostname) + + if err := monitor.Start(ctx, logChan); err != nil { + slog.Error("Error watching tool", "tool", t.Name, "error", err) + } + }) + + slog.Info("Started watching logs", "tool", tool.Name, "file", tool.LogFile) } if cfg.SystemMetrics.Enabled { wg.Add(1) - go func() { + helpers.SafeGo(ctx, "SystemMetrics", func() { defer wg.Done() - collector := NewSystemMetricsCollector(cfg.SystemMetrics, cfg.PollIntervalSeconds) - collector.StartV2(ctx, storage, logChan) - }() + collector := NewSystemMetricsCollector(cfg.SystemMetrics, cfg.PollIntervalSeconds, currentHostname) + collector.Start(ctx, storage, logChan) + }) slog.Info("Started collecting System-Metrics") } if cfg.WebService.Enabled { wg.Add(1) - go func() { + helpers.SafeGo(ctx, "WebService", func() { defer wg.Done() - webService := NewWebServiceV2(cfg, storage) + webService := NewWebService(cfg, storage) if err := webService.Start(ctx); err != nil { - slog.Error("web service error", "error", err) + slog.Error("Web service error", "error", err) } - }() + }) slog.Info("Web service started", "host", cfg.WebService.Host, "port", cfg.WebService.Port) } sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) - <-sigCh - slog.Info("Shutdown-Signal received, stopping threads...") + s := <-sigCh + slog.Info("Shutdown signal received, stopping threads...", "signal", s) cancel() - close(logChan) done := make(chan struct{}) go func() { @@ -181,9 +193,11 @@ func main() { select { case <-done: - slog.Info("All threads closed") + close(logChan) + slog.Info("All threads closed gracefully") case <-time.After(10 * time.Second): - slog.Info("Shutdown-Timeout reached, force quitting") + slog.Error("Shutdown timeout reached, force quitting") + os.Exit(1) } slog.Info("Program stopped") diff --git a/parser/am_parser.go b/parser/am_parser.go deleted file mode 100644 index cfd9950..0000000 --- a/parser/am_parser.go +++ /dev/null @@ -1,49 +0,0 @@ -package parser - -import ( - "log/slog" - "regexp" - "strings" - "time" - "tixel_watch/helpers" - "tixel_watch/models" -) - -var ( - amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`) -) - -type AMParser struct{} - -func (a *AMParser) Parse(line string) (models.LogMessage, error) { - newEntry := models.LogMessage{ - Service: "access-manager", - } - syslogFields, logContent := helpers.ExtractSyslogHeader(line) - newEntry.Host = syslogFields.Hostname - - matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent)) - if len(matches) != 7 { - newEntry.Timestamp = time.Now() - newEntry.LogMessage = line - return newEntry, nil - } - - timestampStr := strings.Join(strings.Split(matches[1], " "), "T") - timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr) - if err != nil { - slog.Error("unable to parse time", "error", err) - return newEntry, err - } - baseInfo := models.AMBaseInfo{ - ProcessID: matches[3], - ThreadID: strings.TrimSpace(matches[4]), - LoggerName: matches[5], - } - newEntry.Timestamp = timestamp - newEntry.LogLevel = matches[2] - newEntry.LogMessage = matches[6] - newEntry.ServiceInformation = baseInfo - - return newEntry, nil -} diff --git a/parser/default_parser.go b/parser/default_parser.go deleted file mode 100644 index 664fc52..0000000 --- a/parser/default_parser.go +++ /dev/null @@ -1,28 +0,0 @@ -package parser - -import ( - "strings" - "time" - "tixel_watch/models" -) - -type DefaultParser struct { - Service string - Tool string -} - -func (d *DefaultParser) Parse(line string) (models.LogMessage, error) { - msg := models.LogMessage{ - LogLevel: "unknown", - LogMessage: strings.TrimSpace(line), - Raw: line, - Timestamp: time.Now(), - } - if d.Service != "" { - msg.Service = d.Service - } - if d.Tool != "" { - msg.Tool = d.Tool - } - return msg, nil -} diff --git a/parser/factory.go b/parser/factory.go index fb6bc15..4293b0f 100644 --- a/parser/factory.go +++ b/parser/factory.go @@ -1,27 +1,10 @@ package parser -func New(serviceName, logType string) (Parser, error) { +func New(serviceName, logType, hostname string) (Parser, error) { switch logType { - case "custom": - switch serviceName { - case "tixstream": - return &TSParser{}, nil - case "transfer-job-manager": - return &TJMParser{}, nil - case "access-manager": - return &arser{}, nil - case "tixel-control-center": - return &TCCParser{}, nil - case "nginx": - return &NginxParser{}, nil - case "nginx-tjm": - return &NginxTJMLogParser{ToolName: serviceName}, nil - default: - return &DefaultParser{Service: serviceName}, nil - } case "json": return &JSONParser{}, nil default: - return &DefaultParser{Service: serviceName}, nil + return NewGenericParser(serviceName, hostname), nil } } diff --git a/parser/generic_parser.go b/parser/generic_parser.go new file mode 100644 index 0000000..af62eba --- /dev/null +++ b/parser/generic_parser.go @@ -0,0 +1,180 @@ +package parser + +import ( + "fmt" + "log/slog" + "strconv" + "strings" + "time" + "watch-tool/models" + "watch-tool/patterns" +) + +type GenericParser struct { + ServiceName string + Hostname string + Extractors []patterns.CompiledExtractor + CommonExt []patterns.CompiledExtractor +} + +func NewGenericParser(serviceName, hostname string) *GenericParser { + repo := patterns.GetInstance() + + var svcExt, commonExt []patterns.CompiledExtractor + if repo != nil { + svcExt = repo.GetExtractors(serviceName) + commonExt = repo.GetExtractors("common") + } else { + slog.Error("CRITICAL: Pattern Repository is nil. Parser will not work correctly.") + } + + return &GenericParser{ + ServiceName: serviceName, + Hostname: hostname, + Extractors: svcExt, + CommonExt: commonExt, + } +} + +func (p *GenericParser) Parse(line string) (models.LogMessage, error) { + entry := models.LogMessage{ + Service: p.ServiceName, + Host: p.Hostname, + Timestamp: time.Now(), + Raw: line, + Fields: make(map[string]any), + Type: "log_entry", + } + + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" { + return entry, nil + } + + allExtractors := append(p.CommonExt, p.Extractors...) + + matchedAny := false + + for _, ext := range allExtractors { + matches := ext.Pattern.FindStringSubmatch(trimmedLine) + if matches == nil { + continue + } + matchedAny = true + + subexpNames := ext.Pattern.SubexpNames() + for i, matchValue := range matches { + if i == 0 { + continue + } + + groupName := subexpNames[i] + if groupName == "" { + continue + } + + cleanValue := strings.TrimSpace(matchValue) + + targetType := ext.Fields[groupName] + parsedValue := p.safeConvert(cleanValue, targetType) + + p.mapField(&entry, groupName, parsedValue) + } + } + + if !matchedAny { + entry.LogMessage = trimmedLine + entry.Fields["_parse_status"] = "failed" + } else if entry.LogMessage == "" { + entry.LogMessage = trimmedLine + } + + return entry, nil +} + +func (p *GenericParser) safeConvert(value, typeDef string) any { + if value == "" || value == "-" { + if strings.HasPrefix(typeDef, "int") || strings.HasPrefix(typeDef, "float") { + return 0 + } + return value + } + + var err error + var result any + + switch { + case strings.HasPrefix(typeDef, "int"): + var i int + i, err = strconv.Atoi(value) + result = i + + case strings.HasPrefix(typeDef, "float"): + var f float64 + f, err = strconv.ParseFloat(value, 64) + result = f + + case strings.HasPrefix(typeDef, "time:"): + layout := strings.TrimPrefix(typeDef, "time:") + result, err = p.parseTimeRobust(value, layout) + + case typeDef == "bool": + var b bool + b, err = strconv.ParseBool(value) + result = b + + default: + return value + } + + if err != nil { + return value + } + + return result +} + +func (p *GenericParser) parseTimeRobust(value, layout string) (time.Time, error) { + if layout == "Jan 02 15:04:05" { + t, err := time.Parse(layout, value) + if err != nil { + return time.Time{}, err + } + now := time.Now() + year := now.Year() + if t.Month() > now.Month() { + year-- + } + return t.AddDate(year, 0, 0), nil + } + + return time.Parse(layout, value) +} + +func (p *GenericParser) mapField(entry *models.LogMessage, key string, value any) { + switch key { + case "timestamp", "time": + if t, ok := value.(time.Time); ok { + entry.Timestamp = t + } + case "log_level", "level": + entry.LogLevel = fmt.Sprintf("%v", value) + case "message", "msg": + entry.LogMessage = fmt.Sprintf("%v", value) + case "host", "hostname": + entry.Host = fmt.Sprintf("%v", value) + case "service": + entry.Service = fmt.Sprintf("%v", value) + case "pid": + if v, ok := value.(int); ok { + entry.PID = v + } else if vStr, ok := value.(string); ok { + if pid, err := strconv.Atoi(vStr); err == nil { + entry.PID = pid + } + } + + default: + entry.Fields[key] = value + } +} diff --git a/parser/json_parser.go b/parser/json_parser.go index fd3527e..d83d068 100644 --- a/parser/json_parser.go +++ b/parser/json_parser.go @@ -3,7 +3,7 @@ package parser import ( "encoding/json" "log/slog" - "tixel_watch/models" + "watch-tool/models" ) type JSONParser struct{} diff --git a/parser/nginx_parser.go b/parser/nginx_parser.go deleted file mode 100644 index ee4da40..0000000 --- a/parser/nginx_parser.go +++ /dev/null @@ -1,57 +0,0 @@ -package parser - -import ( - "log/slog" - "regexp" - "strconv" - "strings" - "tixel_watch/models" -) - -var ( - nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`) -) - -type NginxParser struct{} - -func (n *NginxParser) Parse(line string) (models.LogMessage, error) { - newEntry := models.LogMessage{ - Service: "nginx", - } - - matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(line)) - if len(matches) < 7 { - return newEntry, nil - } - statusCode, err := strconv.ParseInt(matches[5], 10, 64) - if err != nil { - slog.Error("cant parse statuscode", "error", err) - } - bytesSend, err := strconv.ParseInt(matches[6], 10, 64) - if err != nil { - slog.Error("cant parse bytessend", "error", err) - } - baseInfo := models.NGinXBaseInfo{ - ClientIP: matches[1], - RemoteUser: matches[2], - Request: matches[4], - StatusCode: int(statusCode), - BytesSend: int(bytesSend), - } - - if len(matches) > 7 && matches[7] != "" { - baseInfo.Referer = matches[7] - } - if len(matches) > 8 && matches[8] != "" { - baseInfo.UserAgent = matches[8] - } - - if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 { - baseInfo.HTTPMethod = requestParts[0] - baseInfo.RequestURI = requestParts[1] - baseInfo.HTTPVersion = requestParts[2] - } - newEntry.ServiceInformation = baseInfo - - return newEntry, nil -} diff --git a/parser/nginx_tjm_parser.go b/parser/nginx_tjm_parser.go index d5f4f5e..2662a49 100644 --- a/parser/nginx_tjm_parser.go +++ b/parser/nginx_tjm_parser.go @@ -4,12 +4,13 @@ import ( "log/slog" "strconv" "strings" - "tixel_watch/helpers" - "tixel_watch/models" + "watch-tool/helpers" + "watch-tool/models" ) type NginxTJMLogParser struct { ToolName string + Hostname string } func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) { @@ -18,11 +19,7 @@ func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) { Tool: p.ToolName, Raw: line, } - hostname, err := helpers.GetHostname() - if err != nil { - return entry, err - } - entry.Host = hostname + entry.Host = p.Hostname entry = p.parseNginxTJM(entry) return entry, nil } diff --git a/parser/parser.go b/parser/parser.go index 480c741..53d3c63 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -1,11 +1,9 @@ package parser import ( - "tixel_watch/models" + "watch-tool/models" ) type Parser interface { - //TODO: Change parsers to return an error as well Parse(line string) (models.LogMessage, error) - // Parse(line string) models.LogMessage } diff --git a/parser/regex_parser.go b/parser/regex_parser.go deleted file mode 100644 index d12a643..0000000 --- a/parser/regex_parser.go +++ /dev/null @@ -1,64 +0,0 @@ -package parser - -import ( - "log/slog" - "regexp" - "strings" - "tixel_watch/helpers" - "tixel_watch/models" -) - -type RegexLogParser struct { - Pattern *regexp.Regexp - Fields map[string]string - Toolname string -} - -func (p *RegexLogParser) Parse(line string) (models.LogMessage, error) { - entry := models.LogMessage{Type: "log_entry"} - entry.Tool = p.Toolname - entry.Raw = line - hostname, err := helpers.GetHostname() - if err != nil { - slog.Warn("cannot get hostname") - return entry, err - } - entry.Host = hostname - - fields := p.parseWithPattern(line) - if fields != nil { - entry.Fields = fields - } else { - entry.LogMessage = strings.TrimSpace(line) - } - - return entry, nil -} - -func (p *RegexLogParser) parseWithPattern(text string) map[string]any { - matches := p.Pattern.FindStringSubmatch(text) - if matches == nil { - return nil - } - - fields := make(map[string]any) - subexpNames := p.Pattern.SubexpNames() - - for i, match := range matches { - if i == 0 { - continue - } - - if i < len(subexpNames) && subexpNames[i] != "" { - fieldName := subexpNames[i] - - if mappedName, exists := p.Fields[fieldName]; exists { - fieldName = mappedName - } - - fields[fieldName] = match - } - } - - return fields -} diff --git a/parser/tcc_parser.go b/parser/tcc_parser.go deleted file mode 100644 index 2c117c9..0000000 --- a/parser/tcc_parser.go +++ /dev/null @@ -1,49 +0,0 @@ -package parser - -import ( - "log/slog" - "regexp" - "strings" - "time" - "tixel_watch/helpers" - "tixel_watch/models" -) - -var ( - tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`) -) - -type TCCParser struct{} - -func (t *TCCParser) Parse(line string) (models.LogMessage, error) { - newEntry := models.LogMessage{ - Service: "tixel-control-center", - } - syslogFields, logContent := helpers.ExtractSyslogHeader(line) - newEntry.Host = syslogFields.Hostname - - matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(logContent)) - if len(matches) != 7 { - newEntry.Timestamp = time.Now() - newEntry.LogMessage = line - return newEntry, nil - } - - timestampStr := strings.Join(strings.Split(matches[1], " "), "T") - timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr) - if err != nil { - slog.Error("unable to parse time", "error", err) - return newEntry, err - } - baseInfo := models.TCCBaseInfo{ - ProcessID: matches[3], - ThreadID: strings.TrimSpace(matches[4]), - LoggerName: matches[5], - } - newEntry.Timestamp = timestamp - newEntry.LogLevel = matches[2] - newEntry.LogMessage = matches[6] - newEntry.ServiceInformation = baseInfo - - return newEntry, nil -} diff --git a/parser/tjm_parser.go b/parser/tjm_parser.go deleted file mode 100644 index 850e21c..0000000 --- a/parser/tjm_parser.go +++ /dev/null @@ -1,91 +0,0 @@ -package parser - -import ( - "log/slog" - "regexp" - "strings" - "tixel_watch/helpers" - "tixel_watch/models" -) - -var ( - tjmServicePattern = regexp.MustCompile(`^(?