refactor: clean up and add example config and pattern

* dev/implement-drain-parser-package:
  feat: implement drain3 based generic log-parser

build.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-set -e
-PACKAGE_DIR="./packages"
-PACKAGE_NAME="./tixel-watch"
-
-if [ -d "${PACKAGE_DIR}" ]; then
-  rm -rf "${PACKAGE_DIR:?}/"*
-else
-  mkdir -p "${PACKAGE_DIR}"
-fi
-
-if [ -d "${PACKAGE_NAME}" ]; then
-  rm -rf "${PACKAGE_NAME:?}"
-fi
-
-CGO_ENABLED=0 go build -ldflags="-s -w" -o "$PACKAGE_DIR"/tixel-watch -buildvcs .
-
-cp -r ./tixel-watch.service $PACKAGE_DIR/
-cp -r ./configs/ $PACKAGE_DIR/
-cp -r ./install.sh $PACKAGE_DIR/
-mv $PACKAGE_DIR tixel-watch
-
-tar -czvf tixel-watch.tar.gz ./tixel-watch

config.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"fmt"
-	"log"
 	"log/slog"
+	"os"
+	"regexp"
 	"time"
 
 	"github.com/spf13/viper"
@@ -57,8 +58,9 @@ type ElasticsearchConfig struct {
 }
 
 type LocalStorage struct {
-	Enable bool   `mapstructure:"enabled"`
+	Enable         bool                  `mapstructure:"enabled"`
-	DBPath string `mapstructure:"db_path"`
+	DBPath         string                `mapstructure:"db_path"`
+	RotationConfig StorageRotationConfig `mapstructure:"rotation"`
 }
 
 type SystemMetrics struct {
@@ -95,12 +97,88 @@ type Config struct {
 		Level    string `mapstructure:"level"`
 		FilePath string `mapstructure:"file_path"`
 	} `mapstructure:"logging"`
+	PatternsFile string       `mapstructure:"patterns_file"`
+	Drain3       Drain3Config `mapstructure:"drain3"`
+}
+
+type Drain3Config struct {
+	Enabled             bool    `mapstructure:"enabled"`
+	StateDir            string  `mapstructure:"state_dir"`
+	Depth               int     `mapstructure:"depth"`
+	SimThreshold        float64 `mapstructure:"sim_th"`
+	MaxChildren         int     `mapstructure:"max_children"`
+	SaveIntervalSeconds int     `mapstructure:"save_interval"`
+}
+
+type StorageRotationConfig struct {
+	// MaxSizeBytes is the maximum size of the database in bytes (0 = deactivated)
+	MaxSizeBytes int64 `mapstructure:"max_size_bytes"`
+	// MaxAgeHours is the maximum age of the database (0 = deaactivated)
+	MaxAgeHours time.Duration `mapstructure:"max_age_hours"`
+	// MaxFiles is the maximum count of old files, to keep
+	MaxFiles int `mapstructure:"max_files"`
+	// CheckIntervalMinutes is the intervall for checking rotation conditions
+	CheckIntervalMinutes time.Duration `mapstructure:"check_interval_minutes"`
+	// ArchiveDir is the dir to store archived files (empty = same dir as db)
+	ArchiveDir string `mapstructure:"archive_dir"`
+}
+
+func (src StorageRotationConfig) GetMaxAge() time.Duration {
+	if src.MaxAgeHours <= 0 {
+		return 0
+	}
+	return time.Duration(src.MaxAgeHours) * time.Hour
+}
+
+func (src StorageRotationConfig) GetCheckInterval() time.Duration {
+	if src.CheckIntervalMinutes <= 0 {
+		return 5 * time.Minute
+	}
+	return time.Duration(src.CheckIntervalMinutes) * time.Minute
+}
+
+func setConfigDefaults() {
+	viper.SetDefault("poll_interval_seconds", 30)
+	viper.SetDefault("elasticsearch.timeout", 30)
+	viper.SetDefault("system_metrics.enabled", true)
+	viper.SetDefault("system_metrics.collect_cpu", true)
+	viper.SetDefault("system_metrics.collect_memory", true)
+	viper.SetDefault("system_metrics.collect_disk", true)
+	viper.SetDefault("system_metrics.collect_network", false)
+	viper.SetDefault("system_metrics.disk_paths", []string{"/"})
+	viper.SetDefault("web_service.enabled", false)
+	viper.SetDefault("web_service.port", 8080)
+	viper.SetDefault("web_service.host", "localhost")
+	viper.SetDefault("logging.level", "info")
+	viper.SetDefault("export.enabled", true)
+	viper.SetDefault("export.batch_size", 100)
+	viper.SetDefault("export.export_interval", "30s")
+	viper.SetDefault("export.retry_attempts", 3)
+	viper.SetDefault("export.retry_backoff", "5s")
+	viper.SetDefault("export.health_check_interval", "60s")
+	viper.SetDefault("localstorage.enabled", true)
+	viper.SetDefault("localstorage.db_path", "./watch.db")
+	viper.SetDefault("localstorage.rotation.max_size_bytes", int64(100*1024*1024))
+	viper.SetDefault("localstorage.rotation.max_age_hours", 24)
+	viper.SetDefault("localstorage.rotation.max_files", 7)
+	viper.SetDefault("localstorage.rotation.check_interval_minutes", 5)
+	viper.SetDefault("localstorage.rotation.archive_dir", "")
+	viper.SetDefault("patterns_file", "./configs/patterns.yaml")
+	viper.SetDefault("drain3.enabled", true)
+	viper.SetDefault("drain3.state_dir", "./drain3_states")
+	viper.SetDefault("drain3.depth", 4)
+	viper.SetDefault("drain3.sim_th", 0.4)
+	viper.SetDefault("drain3.max_children", 100)
 }
 
 func LoadConfig() (*Config, error) {
+	home, err := os.UserConfigDir()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get user config dir: %w", err)
+	}
 	viper.SetConfigName("config")
 	viper.AddConfigPath(".")
-	viper.AddConfigPath("/opt/tixel/tixel-watch/")
+	viper.AddConfigPath(home)
 	viper.SetConfigType("yaml")
 
 	setConfigDefaults()
@@ -121,92 +199,7 @@ func LoadConfig() (*Config, error) {
 	return &cfg, nil
 }
 
-func setConfigDefaults() {
-	viper.SetDefault("poll_interval_seconds", 30)
-	viper.SetDefault("elasticsearch.timeout", 30)
-	viper.SetDefault("system_metrics.enabled", true)
-	viper.SetDefault("system_metrics.collect_cpu", true)
-	viper.SetDefault("system_metrics.collect_memory", true)
-	viper.SetDefault("system_metrics.collect_disk", true)
-	viper.SetDefault("system_metrics.collect_network", false)
-	viper.SetDefault("system_metrics.disk_paths", []string{"/"})
-	viper.SetDefault("web_service.enabled", false)
-	viper.SetDefault("web_service.port", 8080)
-	viper.SetDefault("web_service.host", "localhost")
-	viper.SetDefault("logging.level", "info")
-}
-
-func setConfigDefaultsV2() {
-	viper.SetDefault("poll_interval_seconds", 30)
-	viper.SetDefault("elasticsearch.timeout", 30)
-	viper.SetDefault("system_metrics.enabled", true)
-	viper.SetDefault("system_metrics.collect_cpu", true)
-	viper.SetDefault("system_metrics.collect_memory", true)
-	viper.SetDefault("system_metrics.collect_disk", true)
-	viper.SetDefault("system_metrics.collect_network", false)
-	viper.SetDefault("system_metrics.disk_paths", []string{"/"})
-	viper.SetDefault("web_service.enabled", false)
-	viper.SetDefault("web_service.port", 8080)
-	viper.SetDefault("web_service.host", "localhost")
-	viper.SetDefault("logging.level", "info")
-	viper.SetDefault("export.enabled", true)
-	viper.SetDefault("export.batch_size", 100)
-	viper.SetDefault("export.export_interval", "30s")
-	viper.SetDefault("export.retry_attempts", 3)
-	viper.SetDefault("export.retry_backoff", "5s")
-	viper.SetDefault("export.health_check_interval", "60s")
-	viper.SetDefault("localstorage.enabled", true)
-	viper.SetDefault("localstorage.db_path", "./tixel_watch.db")
-}
-
 func validateConfig(cfg *Config) error {
-	if cfg.Elasticsearch.URL == "" {
-		return fmt.Errorf("elasticsearch.url is required")
-	}
-
-	if cfg.Elasticsearch.Index == "" {
-		return fmt.Errorf("elasticsearch.index is required")
-	}
-
-	if cfg.PollIntervalSeconds <= 0 {
-		log.Printf("Warning: poll_interval_seconds is %d, setting to 30", cfg.PollIntervalSeconds)
-		cfg.PollIntervalSeconds = 30
-	}
-
-	for i := range cfg.Tools {
-		if cfg.Tools[i].BufferSize <= 0 {
-			cfg.Tools[i].BufferSize = 100
-		}
-	}
-
-	return nil
-}
-
-func LoadConfigV2() (*Config, error) {
-	viper.SetConfigName("config")
-	viper.AddConfigPath(".")
-	viper.AddConfigPath("/opt/tixel/tixel-watch/")
-	viper.SetConfigType("yaml")
-
-	setConfigDefaultsV2()
-
-	if err := viper.ReadInConfig(); err != nil {
-		return nil, fmt.Errorf("error reading config: %w", err)
-	}
-
-	var cfg Config
-	if err := viper.Unmarshal(&cfg); err != nil {
-		return nil, fmt.Errorf("error parsing config: %w", err)
-	}
-
-	if err := validateConfigV2(&cfg); err != nil {
-		return nil, fmt.Errorf("config validation failed: %w", err)
-	}
-
-	return &cfg, nil
-}
-
-func validateConfigV2(cfg *Config) error {
 	if !cfg.LocalStorage.Enable {
 		return fmt.Errorf("local storage must be enabled in the new architecture")
 	}
@@ -247,10 +240,36 @@ func validateConfigV2(cfg *Config) error {
 		}
 	}
 
-	for i := range cfg.Tools {
+	for _, tool := range cfg.Tools {
-		if cfg.Tools[i].BufferSize <= 0 {
+		if tool.BufferSize <= 0 {
-			cfg.Tools[i].BufferSize = 100
+			tool.BufferSize = 100
 		}
+
+		if tool.Format.Pattern != "" {
+			if _, err := regexp.Compile(tool.Format.Pattern); err != nil {
+				return fmt.Errorf("invalid regex for tool '%s': %w", tool.Name, err)
+			}
+		}
+	}
+
+	if cfg.LocalStorage.RotationConfig.MaxSizeBytes < 0 {
+		slog.Warn("Invalid rotation max_size_bytes, setting to 100MB", "value", cfg.LocalStorage.RotationConfig.MaxSizeBytes)
+		cfg.LocalStorage.RotationConfig.MaxSizeBytes = 100 * 1024 * 1024
+	}
+
+	if cfg.LocalStorage.RotationConfig.MaxAgeHours < 0 {
+		slog.Warn("Invalid rotation max_age_hours, setting to 24", "value", cfg.LocalStorage.RotationConfig.MaxAgeHours)
+		cfg.LocalStorage.RotationConfig.MaxAgeHours = 24
+	}
+
+	if cfg.LocalStorage.RotationConfig.MaxFiles < 0 {
+		slog.Warn("Invalid rotation max_files, setting to 7", "value", cfg.LocalStorage.RotationConfig.MaxFiles)
+		cfg.LocalStorage.RotationConfig.MaxFiles = 7
+	}
+
+	if cfg.LocalStorage.RotationConfig.CheckIntervalMinutes < 1 {
+		slog.Warn("Invalid rotation check_interval_minutes, setting to 5", "value", cfg.LocalStorage.RotationConfig.CheckIntervalMinutes)
+		cfg.LocalStorage.RotationConfig.CheckIntervalMinutes = 5
 	}
 
 	return nil

configs/elasticsearch.yml

@@ -1,119 +0,0 @@
-# ======================== Elasticsearch Configuration =========================
-#
-# NOTE: Elasticsearch comes with reasonable defaults for most settings.
-#       Before you set out to tweak and tune the configuration, make sure you
-#       understand what are you trying to accomplish and the consequences.
-#
-# The primary way of configuring a node is via this file. This template lists
-# the most important settings you may want to configure for a production cluster.
-#
-# Please consult the documentation for further information on configuration options:
-# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
-#
-# ---------------------------------- Cluster -----------------------------------
-#
-# Use a descriptive name for your cluster:
-#
-cluster.name: tixel-elastic
-#
-# ------------------------------------ Node ------------------------------------
-#
-# Use a descriptive name for the node:
-#
-#node.name: node-1
-#
-# Add custom attributes to the node:
-#
-#node.attr.rack: r1
-#
-# ----------------------------------- Paths ------------------------------------
-#
-# Path to directory where to store the data (separate multiple locations by comma):
-#
-path.data: /var/lib/elasticsearch
-#
-# Path to log files:
-#
-path.logs: /var/log/elasticsearch
-#
-# ----------------------------------- Memory -----------------------------------
-#
-# Lock the memory on startup:
-#
-#bootstrap.memory_lock: true
-#
-# Make sure that the heap size is set to about half the memory available
-# on the system and that the owner of the process is allowed to use this
-# limit.
-#
-# Elasticsearch performs poorly when the system is swapping the memory.
-#
-# ---------------------------------- Network -----------------------------------
-#
-# By default Elasticsearch is only accessible on localhost. Set a different
-# address here to expose this node on the network:
-#
-network.host: 0.0.0.0
-#
-# By default Elasticsearch listens for HTTP traffic on the first free port it
-# finds starting at 9200. Set a specific HTTP port here:
-#
-#http.port: 9200
-#
-# For more information, consult the network module documentation.
-#
-# --------------------------------- Discovery ----------------------------------
-#
-# Pass an initial list of hosts to perform discovery when this node is started:
-# The default list of hosts is ["127.0.0.1", "[::1]"]
-#
-#discovery.seed_hosts: ["host1", "host2"]
-#
-# Bootstrap the cluster using an initial set of master-eligible nodes:
-#
-#cluster.initial_master_nodes: ["node-1", "node-2"]
-#
-# For more information, consult the discovery and cluster formation module documentation.
-#
-# ---------------------------------- Various -----------------------------------
-#
-# Allow wildcard deletion of indices:
-#
-#action.destructive_requires_name: false
-
-#----------------------- BEGIN SECURITY AUTO CONFIGURATION -----------------------
-#
-# The following settings, TLS certificates, and keys have been automatically      
-# generated to configure Elasticsearch security features on 26-08-2025 14:51:23
-#
-# --------------------------------------------------------------------------------
-
-# Enable security features
-xpack.security.enabled: false
-
-xpack.security.enrollment.enabled: false
-
-# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents
-xpack.security.http.ssl:
-  enabled: false
-  keystore.path: certs/http.p12
-
-# Enable encryption and mutual authentication between cluster nodes
-xpack.security.transport.ssl:
-  enabled: false
-  verification_mode: certificate
-  keystore.path: certs/transport.p12
-  truststore.path: certs/transport.p12
-# Create a new cluster with the current node only
-# Additional nodes can still join the cluster later
-cluster.initial_master_nodes: ["frankfurt.tixeltec.de"]
-
-# Allow HTTP API connections from anywhere
-# Connections are encrypted and require user authentication
-http.host: 0.0.0.0
-
-# Allow other nodes to join the cluster from anywhere
-# Connections are encrypted and mutually authenticated
-transport.host: 0.0.0.0
-
-#----------------------- END SECURITY AUTO CONFIGURATION -------------------------

configs/example-config.yaml

@@ -1,35 +1,74 @@
+
+export:
+  enabled: true
+  batch_size: 100
+  export_interval: "30s"
+  retry_attempts: 5
+  retry_backoff: "10s"
+  health_check_interval: "60s"
+
+localstorage:
+  enabled: true
+  db_path: "./watch.db"
+  rotation:
+    max_sizes_bytes: 100 * 1024 * 1024
+    max_age_hours: 24
+    max_files: 3
+    check_interval_minuntes: 5
+    archive_dir: ""
+
 elasticsearch:
-  url: "http://localhost:9200"
+  enabled: true
-  index: "tixel"
+  url: "http://10.0.0.99:9200"
-  username: "elastic"
+  index: "watch"
-  password: "79QQ4JGTa3R_nkqA=MxW"
+  username: "your-configured-user"
+  password: "your-super-secret-password"
+  api_key: "your-api-key"
   timeout: 30
 
 web_service:
   enabled: true
-  host: "localhost"
+  host: "0.0.0.0"
-  port: 8080
+  port: 9090
 
 system_metrics:
   enabled: true
   collect_cpu: true
   collect_memory: true
   collect_disk: true
-  collect_network: false
+  collect_network: true
   disk_paths:
     - "/"
     - "/var"
     - "/home"
   network_interfaces:
-    - "eth0"
+    - "ens6"
-    - "wlan0"
+  collect_network_connections: true
+  collect_load_average: true
+  collect_tcp_stats: true
+  collect_filehandles: true
+  collect_disk_io: true
+  collect_network_latency: true
+  collect_bandwidth_usage: true
+  transfer_ports: 60003
+  latency_test_hosts: "www.google.de"
 
 poll_interval_seconds: 30
+patterns_file: "./configs/patterns.yaml"
 
 logging:
   level: "info"
   file_path: "/var/log/system-monitor.log"
 
+drain3:
+  enabled: true
+  state_dir: "./drain3_states"
+  depth: 4
+  sim_th: 0.4
+  max_children: 100
+  max_clusters: 1000
+  save_interval: 60
+
 services:
   - name: "nginx"
     service: "nginx.service"
@@ -37,18 +76,6 @@ services:
     since_time: ""
     priority: "info"
 
-  - name: "tixstream"
-    service: "tixstream.service"
-    enabled: true
-    since_time: ""
-    priority: "debug"
-
-  - name: "transfer-job-manager"
-    service: "transfer-job-manager.service"
-    enabled: true
-    since_time: ""
-    priority: "debug"
-
 tools:
   - name: "nginx-access"
     log_file: "/var/log/nginx/access.log"
@@ -82,9 +109,3 @@ tools:
         tid: "thread_id"
         message: "error_message"
 
-  - name: "nginx-tjm"
-    log_file: "/var/log/nginx/access_tjm.log"
-    enabled: true
-    buffer_size: 100
-    format:
-      name: "custom"

configs/example-patterns.yaml

@@ -0,0 +1,36 @@
+patterns:
+  common:
+    extractors:
+      - name: "syslog_header"
+        regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+) (?P<process_info>[^:]+):\s*(?P<message_rest>.*)$'
+        fields:
+          syslog_timestamp: "time:Jan 02 15:04:05"
+          hostname: "string"
+          process_info: "string"
+          message_rest: "string"
+      
+      - name: "iso8601_timestamp"
+        regex: '(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)'
+        fields:
+          timestamp: "time:2006-01-02T15:04:05.000000Z"
+
+  nginx:
+    extractors:
+      - name: "access_log"
+        regex: '^(?P<client_ip>\S+)\s+\S+\s+(?P<remote_user>\S+)\s+\[(?P<timestamp_nginx>[^\]]+)\]\s+"(?P<request>[^"]+)"\s+(?P<status_code>\d+)\s+(?P<bytes_sent>\d+|-)'
+        fields:
+          client_ip: "string"
+          remote_user: "string"
+          timestamp_nginx: "string"
+          request: "string"
+          status_code: "int"
+          bytes_sent: "int"
+
+  my-app:
+    extractors:
+      - name: "app_log"
+        regex: '^\[(?P<level>\w+)\] id=(?P<request_id>\d+) duration=(?P<duration_ms>\d+)ms'
+        fields:
+          level: "string"
+          request_id: "int"
+          duration_ms: "int"

elasticsearch.go

@@ -7,7 +7,7 @@ import (
 	"log/slog"
 	"strings"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
 
 	"github.com/elastic/go-elasticsearch/v8"
 )
@@ -70,8 +70,7 @@ func (esc *ElasticsearchClient) SendBatch(baseIndex string, entries []models.Log
 
 	var body strings.Builder
 	for _, entry := range entries {
-		// indexName := determineIndexName(baseIndex, entry)
+		indexName := "watch"
-		indexName := "tixel"
 
 		indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName)
 		body.WriteString(indexLine)
@@ -118,7 +117,7 @@ func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics mode
 		return fmt.Errorf("JSON marshalling error: %w", err)
 	}
 
-	systemIndex := "tixel"
+	systemIndex := "watch"
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -146,14 +145,3 @@ func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics mode
 
 	return nil
 }
-
-// func determineIndexName(baseIndex string, entry LogEntry) string {
-// 	switch entry.Type {
-// 	case "system_metrics":
-// 		return fmt.Sprintf("%s-system", baseIndex)
-// 	case "service_log":
-// 		return fmt.Sprintf("%s-service-%s", baseIndex, entry.Service)
-// 	default:
-// 		return fmt.Sprintf("%s-%s", baseIndex, entry.Tool)
-// 	}
-// }

elasticsearch_exporter.go

@@ -4,280 +4,87 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"log/slog"
 	"strings"
 	"time"
+	"watch-tool/models"
 
 	"github.com/elastic/go-elasticsearch/v8"
 )
 
 type ElasticsearchExporter struct {
 	client *elasticsearch.Client
+	config ElasticsearchConfig
 }
 
-func NewElasticsearchExporter(client *elasticsearch.Client) *ElasticsearchExporter {
+func NewElasticsearchExporter(config ElasticsearchConfig) (*ElasticsearchExporter, error) {
+	client, err := NewElasticsearchClient(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err)
+	}
+
 	return &ElasticsearchExporter{
 		client: client,
-	}
+		config: config,
+	}, nil
 }
 
-type ExportResult struct {
+func (e *ElasticsearchExporter) Export(ctx context.Context, entries []models.LogMessage) error {
-	Index         string    `json:"index"`
+	if len(entries) == 0 {
-	DocumentCount int       `json:"document_count"`
+		return nil
-	StartTime     time.Time `json:"start_time"`
-	EndTime       time.Time `json:"end_time"`
-	Duration      string    `json:"duration"`
-	Error         string    `json:"error,omitempty"`
-}
-
-func (e *ElasticsearchExporter) ExportToStream(ctx context.Context, indices []string, batchSize int, since int, writer io.Writer) error {
-	startTime := time.Now()
-
-	if _, err := writer.Write([]byte("{\n  \"export_info\": {\n")); err != nil {
-		return fmt.Errorf("error writing export header: %w", err)
 	}
 
-	exportInfo := map[string]any{
+	var body strings.Builder
-		"timestamp":  startTime,
+	for _, entry := range entries {
-		"indices":    indices,
+		indexName := e.config.Index
-		"batch_size": batchSize,
-		"sinceDays":  since,
-	}
 
-	infoBytes, err := json.MarshalIndent(exportInfo, "    ", "  ")
+		indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName)
-	if err != nil {
+		body.WriteString(indexLine)
-		return fmt.Errorf("error marshalling export info: %w", err)
+		body.WriteString("\n")
-	}
 
-	infoStr := string(infoBytes)
+		data, err := json.Marshal(entry)
-	infoStr = strings.TrimPrefix(infoStr, "{")
+		if err != nil {
-	infoStr = strings.TrimSuffix(infoStr, "}")
+			slog.Error("error marshalling JSON", "error", err)
-
+			continue
-	if _, err := writer.Write([]byte(infoStr)); err != nil {
-		return fmt.Errorf("error writing export info: %w", err)
-	}
-
-	if _, err := writer.Write([]byte("\n  },\n  \"data\": {\n")); err != nil {
-		return fmt.Errorf("error writing data header: %w", err)
-	}
-
-	results := make([]ExportResult, 0, len(indices))
-	first := true
-
-	for _, index := range indices {
-		if !first {
-			if _, err := writer.Write([]byte(",\n")); err != nil {
-				return fmt.Errorf("error writing separator: %w", err)
-			}
-		}
-		first = false
-
-		result := e.exportIndex(ctx, index, batchSize, since, writer)
-		results = append(results, result)
-
-		if result.Error != "" {
-			slog.Error("error exporting index", "index", index, "error", result.Error)
 		}
+		body.WriteString(string(data))
+		body.WriteString("\n")
 	}
 
-	if _, err := writer.Write([]byte("\n  },\n  \"results\": ")); err != nil {
+	timeout := time.Duration(e.config.Timeout) * time.Second
-		return fmt.Errorf("error writing results header: %w", err)
+	ctx, cancel := context.WithTimeout(ctx, timeout)
-	}
+	defer cancel()
 
-	if err := json.NewEncoder(writer).Encode(results); err != nil {
+	res, err := e.client.Bulk(
-		return fmt.Errorf("error writing results: %w", err)
+		strings.NewReader(body.String()),
-	}
+		e.client.Bulk.WithContext(ctx),
-
-	if _, err := writer.Write([]byte("}\n")); err != nil {
-		return fmt.Errorf("error writing final bracket: %w", err)
-	}
-
-	duration := time.Since(startTime)
-	slog.Info("Export completed", "duration", duration, "indices_count", len(indices))
-
-	return nil
-}
-
-func (e *ElasticsearchExporter) exportIndex(ctx context.Context, index string, batchSize int, since int, writer io.Writer) ExportResult {
-	startTime := time.Now()
-	result := ExportResult{
-		Index:     index,
-		StartTime: startTime,
-	}
-
-	if _, err := fmt.Fprintf(writer, "    \"%s\": [\n", index); err != nil {
-		result.Error = fmt.Sprintf("error writing index header: %v", err)
-		result.EndTime = time.Now()
-		result.Duration = time.Since(startTime).String()
-		return result
-	}
-
-	query := `{"query":{"match_all":{}}}`
-	if since > 0 {
-		query = fmt.Sprintf(`{
-  "query": {
-    "range": {
-      "timestamp": {
-        "gte": "now-%dd/d",
-        "lt": "now/d"
-      }
-    }
-  }
-}`, since)
-	}
-	res, err := e.client.Search(
-		e.client.Search.WithContext(ctx),
-		e.client.Search.WithIndex(index),
-		e.client.Search.WithScroll(1000),
-		e.client.Search.WithSize(batchSize),
-		e.client.Search.WithBody(strings.NewReader(query)),
 	)
 	if err != nil {
-		result.Error = fmt.Sprintf("error in initial search: %v", err)
+		return fmt.Errorf("bulk request error: %w", err)
-		result.EndTime = time.Now()
-		result.Duration = time.Since(startTime).String()
-		return result
 	}
 	defer res.Body.Close()
 
 	if res.IsError() {
-		result.Error = fmt.Sprintf("elasticsearch error: %s", res.String())
+		return fmt.Errorf("bulk request failed: %s", res.String())
-		result.EndTime = time.Now()
-		result.Duration = time.Since(startTime).String()
-		return result
 	}
 
-	var searchResult map[string]any
+	slog.Debug("Batch successfully exported to Elasticsearch", "count", len(entries))
-	if err := json.NewDecoder(res.Body).Decode(&searchResult); err != nil {
+	return nil
-		result.Error = fmt.Sprintf("error decoding search result: %v", err)
-		result.EndTime = time.Now()
-		result.Duration = time.Since(startTime).String()
-		return result
-	}
-
-	scrollID, ok := searchResult["_scroll_id"].(string)
-	if !ok {
-		result.Error = "no scroll_id found in search result"
-		result.EndTime = time.Now()
-		result.Duration = time.Since(startTime).String()
-		return result
-	}
-
-	hits := searchResult["hits"].(map[string]any)["hits"].([]any)
-	firstDocument := true
-	documentCount := 0
-
-	for _, hit := range hits {
-		if !firstDocument {
-			if _, err := writer.Write([]byte(",\n")); err != nil {
-				result.Error = fmt.Sprintf("error writing separator: %v", err)
-				result.EndTime = time.Now()
-				result.Duration = time.Since(startTime).String()
-				return result
-			}
-		}
-		firstDocument = false
-
-		source := hit.(map[string]any)["_source"]
-		if err := e.writeDocument(writer, source); err != nil {
-			result.Error = fmt.Sprintf("error writing document: %v", err)
-			result.EndTime = time.Now()
-			result.Duration = time.Since(startTime).String()
-			return result
-		}
-		documentCount++
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			result.Error = "context cancelled"
-			result.EndTime = time.Now()
-			result.Duration = time.Since(startTime).String()
-			return result
-		default:
-		}
-
-		scrollRes, err := e.client.Scroll(
-			e.client.Scroll.WithScrollID(scrollID),
-			e.client.Scroll.WithScroll(1000),
-			e.client.Scroll.WithContext(ctx),
-		)
-		if err != nil {
-			result.Error = fmt.Sprintf("error in scroll request: %v", err)
-			break
-		}
-		defer scrollRes.Body.Close()
-
-		if scrollRes.IsError() {
-			result.Error = fmt.Sprintf("elasticsearch scroll error: %s", scrollRes.String())
-			break
-		}
-
-		var scrollResult map[string]any
-		if err := json.NewDecoder(scrollRes.Body).Decode(&scrollResult); err != nil {
-			result.Error = fmt.Sprintf("error decoding scroll result: %v", err)
-			break
-		}
-
-		hits := scrollResult["hits"].(map[string]any)["hits"].([]any)
-		if len(hits) == 0 {
-			break
-		}
-
-		scrollID, _ = scrollResult["_scroll_id"].(string)
-
-		for _, hit := range hits {
-			if _, err := writer.Write([]byte(",\n")); err != nil {
-				result.Error = fmt.Sprintf("error writing separator: %v", err)
-				result.EndTime = time.Now()
-				result.Duration = time.Since(startTime).String()
-				return result
-			}
-
-			source := hit.(map[string]any)["_source"]
-			if err := e.writeDocument(writer, source); err != nil {
-				result.Error = fmt.Sprintf("error writing document: %v", err)
-				result.EndTime = time.Now()
-				result.Duration = time.Since(startTime).String()
-				return result
-			}
-			documentCount++
-		}
-	}
-
-	if _, err := writer.Write([]byte("\n    ]")); err != nil {
-		if result.Error == "" {
-			result.Error = fmt.Sprintf("error writing index footer: %v", err)
-		}
-	}
-
-	result.DocumentCount = documentCount
-	result.EndTime = time.Now()
-	result.Duration = time.Since(startTime).String()
-
-	slog.Info("Index export completed",
-		"index", index,
-		"documents", documentCount,
-		"duration", result.Duration,
-	)
-
-	return result
 }
 
-func (e *ElasticsearchExporter) writeDocument(writer io.Writer, document any) error {
+func (e *ElasticsearchExporter) HealthCheck(ctx context.Context) error {
-	jsonBytes, err := json.MarshalIndent(document, "      ", "  ")
+	timeout := time.Duration(e.config.Timeout) * time.Second
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	res, err := e.client.Info(e.client.Info.WithContext(ctx))
 	if err != nil {
-		return fmt.Errorf("error marshalling document: %w", err)
+		return fmt.Errorf("health check failed: %w", err)
 	}
+	defer res.Body.Close()
 
-	if _, err := writer.Write([]byte("      ")); err != nil {
+	if res.IsError() {
-		return err
+		return fmt.Errorf("health check failed: %s", res.String())
-	}
-
-	if _, err := writer.Write(jsonBytes); err != nil {
-		return err
 	}
 
 	return nil

elasticsearch_exporterV2.go

@@ -1,91 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"log/slog"
-	"strings"
-	"time"
-	"tixel_watch/models"
-
-	"github.com/elastic/go-elasticsearch/v8"
-)
-
-type ElasticsearchExporterV2 struct {
-	client *elasticsearch.Client
-	config ElasticsearchConfig
-}
-
-func NewElasticsearchExporterV2(config ElasticsearchConfig) (*ElasticsearchExporterV2, error) {
-	client, err := NewElasticsearchClient(config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err)
-	}
-
-	return &ElasticsearchExporterV2{
-		client: client,
-		config: config,
-	}, nil
-}
-
-func (e *ElasticsearchExporterV2) Export(ctx context.Context, entries []models.LogMessage) error {
-	if len(entries) == 0 {
-		return nil
-	}
-
-	var body strings.Builder
-	for _, entry := range entries {
-		indexName := e.config.Index
-
-		indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName)
-		body.WriteString(indexLine)
-		body.WriteString("\n")
-
-		data, err := json.Marshal(entry)
-		if err != nil {
-			slog.Error("error marshalling JSON", "error", err)
-			continue
-		}
-		body.WriteString(string(data))
-		body.WriteString("\n")
-	}
-
-	timeout := time.Duration(e.config.Timeout) * time.Second
-	ctx, cancel := context.WithTimeout(ctx, timeout)
-	defer cancel()
-
-	res, err := e.client.Bulk(
-		strings.NewReader(body.String()),
-		e.client.Bulk.WithContext(ctx),
-	)
-	if err != nil {
-		return fmt.Errorf("bulk request error: %w", err)
-	}
-	defer res.Body.Close()
-
-	if res.IsError() {
-		return fmt.Errorf("bulk request failed: %s", res.String())
-	}
-
-	slog.Debug("Batch successfully exported to Elasticsearch", "count", len(entries))
-	return nil
-}
-
-func (e *ElasticsearchExporterV2) HealthCheck(ctx context.Context) error {
-	timeout := time.Duration(e.config.Timeout) * time.Second
-	ctx, cancel := context.WithTimeout(ctx, timeout)
-	defer cancel()
-
-	res, err := e.client.Info(e.client.Info.WithContext(ctx))
-	if err != nil {
-		return fmt.Errorf("health check failed: %w", err)
-	}
-	defer res.Body.Close()
-
-	if res.IsError() {
-		return fmt.Errorf("health check failed: %s", res.String())
-	}
-
-	return nil
-}

export_manager.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 type ExportManager struct {

exporter_interface.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"context"
+	"watch-tool/models"
+)
+
+type ExporterInterface interface {
+	Export(ctx context.Context, entries []models.LogMessage) error
+	HealthCheck(ctx context.Context) error
+}

file_monitor.go

@@ -6,47 +6,67 @@ import (
 	"log/slog"
 	"regexp"
 	"strings"
-	"tixel_watch/models"
+	"watch-tool/models"
-	"tixel_watch/parser"
+	"watch-tool/parser"
+	"watch-tool/patterns"
 
+	"codeberg.org/pata1704/drain3"
 	"github.com/hpcloud/tail"
 )
 
 type FileMonitor struct {
-	config ToolConfig
+	config   ToolConfig
-	parser parser.Parser
+	parser   parser.Parser
+	hostname string
 }
 
-func NewFileMonitor(config ToolConfig) *FileMonitor {
+func NewFileMonitor(config ToolConfig, hostname string, drainCfg *drain3.Config, stateDir string) *FileMonitor {
 	var logParser parser.Parser
 
+	pCfg := parser.ParserConfig{
+		ServiceName: config.Name,
+		LogType:     "custom",
+		Hostname:    hostname,
+		DrainConfig: drainCfg,
+		StateDir:    stateDir,
+	}
+
 	if config.Format.Pattern != "" {
-		pattern, err := regexp.Compile(config.Format.Pattern)
+		compiledRegex, err := regexp.Compile(config.Format.Pattern)
 		if err != nil {
-			slog.Error("invalid regex pattern", "tool", config.Name, "error", err)
+			slog.Error("Invalid regex pattern in tool config", "tool", config.Name, "error", err)
-			logParser = &parser.DefaultParser{}
+			logParser = parser.NewGenericParser(config.Name, hostname, pCfg.DrainConfig, pCfg.StateDir)
+
 		} else {
-			logParser = &parser.RegexLogParser{
+			gp := parser.NewGenericParser(config.Name, hostname, pCfg.DrainConfig, pCfg.StateDir)
-				Pattern:  pattern,
+
-				Fields:   config.Format.Fields,
+			customExtractor := patterns.CompiledExtractor{
-				Toolname: config.Name,
+				Name:    "config_custom_pattern",
+				Pattern: compiledRegex,
+				Fields:  config.Format.Fields,
 			}
+
+			gp.Extractors = append(gp.Extractors, customExtractor)
+			logParser = gp
 		}
 	} else {
 		var err error
-		logParser, err = parser.New(config.Name, "custom")
+		logParser, err = parser.New(pCfg)
 		if err != nil {
-			slog.Error("cannot get tool specific parser", "error", err)
+			slog.Error("Cannot get tool specific parser from factory", "error", err)
+			logParser = parser.NewGenericParser(config.Name, hostname, pCfg.DrainConfig, pCfg.StateDir)
 		}
 	}
 
 	return &FileMonitor{
-		config: config,
+		config:   config,
-		parser: logParser,
+		parser:   logParser,
+		hostname: hostname,
 	}
 }
 
 func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage) error {
+	defer fm.parser.Close()
 	t, err := tail.TailFile(fm.config.LogFile, tail.Config{
 		Follow:    true,
 		ReOpen:    true,
@@ -72,7 +92,7 @@ func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage)
 			}
 
 			if line.Err != nil {
-				slog.Error("error reading log file", "tool", fm.config.Name, "error", line.Err)
+				slog.Error("Error reading log file", "tool", fm.config.Name, "error", line.Err)
 				continue
 			}
 
@@ -82,7 +102,11 @@ func (fm *FileMonitor) Start(ctx context.Context, out chan<- models.LogMessage)
 
 			entry, err := fm.parser.Parse(line.Text)
 			if err != nil {
-				slog.Error("error parsing log line", "error", err)
+				slog.Error("Error parsing log line", "tool", fm.config.Name, "error", err)
+			} else {
+				if entry.Tool == "" {
+					entry.Tool = fm.config.Name
+				}
 			}
 
 			select {

go.mod

@@ -1,14 +1,16 @@
-module tixel_watch
+module watch-tool
 
-go 1.24.1
+go 1.25.5
 
 require (
+	codeberg.org/pata1704/drain3 v1.0.0
 	github.com/elastic/go-elasticsearch/v8 v8.19.0
 	github.com/hpcloud/tail v1.0.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/viper v1.20.1
-	golang.org/x/sys v0.34.0
+	golang.org/x/sys v0.37.0
-	modernc.org/sqlite v1.39.0
+	gopkg.in/yaml.v3 v3.0.1
+	modernc.org/sqlite v1.44.1
 )
 
 require (
@@ -21,7 +23,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
@@ -38,12 +40,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
+	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.67.6 // indirect
-	modernc.org/libc v1.66.3 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+codeberg.org/pata1704/drain3 v1.0.0 h1:X66fn+lnzOMU+PFFSkNBF89z1ghbqihE1I4A6x/OJIM=
+codeberg.org/pata1704/drain3 v1.0.0/go.mod h1:+K1hIYh3hNSPiXRxUin6ZiC2CC9FDGqQKNNR+7ZIx9s=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -26,6 +28,8 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17k
 github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -34,8 +38,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
-github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -82,20 +86,20 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
-golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
+golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
-golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
+golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
-golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
+golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
-golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
+golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
+golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -105,18 +109,20 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM=
+modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
-modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
-modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
+modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc=
-modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
+modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM=
-modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM=
+modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
-modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
+modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
 modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
 modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE=
+modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
 modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
 modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
-modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
+modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI=
-modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
+modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE=
 modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
 modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
 modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
@@ -125,8 +131,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
 modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
 modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
-modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY=
+modernc.org/sqlite v1.44.1 h1:qybx/rNpfQipX/t47OxbHmkkJuv2JWifCMH8SVUiDas=
-modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
+modernc.org/sqlite v1.44.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
 modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
 modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

helpers/helpers.go

@@ -3,11 +3,10 @@ package helpers
 import (
 	"fmt"
 	"log/slog"
-	"os"
 	"regexp"
 	"strings"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 var (
@@ -76,11 +75,3 @@ func ParseSyslogTimeToRFC3339(syslogTime string) (time.Time, error) {
 	t = t.AddDate(now.Year(), 0, 0)
 	return t, nil
 }
-
-func GetHostname() (string, error) {
-	hostname, err := os.Hostname()
-	if err != nil {
-		hostname = "unknown"
-	}
-	return hostname, nil
-}

helpers/utils.go

@@ -0,0 +1,47 @@
+package helpers
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"runtime/debug"
+)
+
+type AppError struct {
+	Op      string
+	Err     error
+	Context string
+}
+
+func (e *AppError) Error() string {
+	if e.Context != "" {
+		return fmt.Sprintf("%s: %v (%s)", e.Op, e.Err, e.Context)
+	}
+	return fmt.Sprintf("%s: %v", e.Op, e.Err)
+}
+
+func (e *AppError) Unwrap() error {
+	return e.Err
+}
+
+func NewAppError(op string, err error, ctx string) error {
+	return &AppError{Op: op, Err: err, Context: ctx}
+}
+
+func SafeGo(ctx context.Context, name string, fn func()) {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				stack := string(debug.Stack())
+				slog.Error("CRITICAL: Panic recovered in goroutine",
+					"goroutine", name,
+					"panic", r,
+					"stack", stack,
+				)
+				// Optional: Hier könnte man Metriken inkrementieren (siehe Observability)
+			}
+		}()
+
+		fn()
+	}()
+}

install.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-set -e
-
-ES_VERSION="9.1.2"
-ES_DEB_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-amd64.deb"
-ES_RPM_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-x86_64.rpm"
-ES_CONFIG_DIR="/etc/elasticsearch"
-ES_JVM_OPTIONS="/etc/elasticsearch/jvm.options"
-ES_JVM_OPTIONS_D="/etc/elasticsearch/jvm.options.d"
-GO_SERVICE_NAME="tixel-watch"
-GO_INSTALL_TARGET="/opt/tixel/tixel-watch"
-
-install_es_deb() {
-  echo "Installing Elasticsearch (Debian package)..."
-  wget "${ES_DEB_URL}" -O elasticsearch.deb
-  sudo dpkg -i elasticsearch.deb
-  sudo apt-get install -f -y
-}
-
-install_es_rpm() {
-  echo "Installing Elasticsearch (RPM package)..."
-  wget "${ES_RPM_URL}" -O elasticsearch.rpm
-  sudo rpm --install elasticsearch.rpm
-}
-
-setup_configuration() {
-  echo "Copying Elasticsearch configuration files..."
-  sudo cp ./configs/elasticsearch.yml "${ES_CONFIG_DIR}/elasticsearch.yml"
-  sudo cp ./configs/jvm.options "${ES_JVM_OPTIONS}"
-  sudo cp -r ./configs/jvm.options.d "${ES_JVM_OPTIONS_D}"
-  sudo chown root:elasticsearch "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}"
-  sudo chmod 640 "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}"
-}
-
-setup_tixel_watch_service() {
-  echo "Setting up tixel-watch systemd service..."
-  if [ ! -d ${GO_INSTALL_TARGET} ]; then
-    mkdir -p ${GO_INSTALL_TARGET}
-  fi
-  sudo cp ./tixel-watch "$GO_INSTALL_TARGET"/
-  sudo cp ./configs/config.yaml "$GO_INSTALL_TARGET"/
-  sudo cp ./${GO_SERVICE_NAME}.service /etc/systemd/system/${GO_SERVICE_NAME}.service
-  sudo systemctl daemon-reload
-  sudo systemctl enable "${GO_SERVICE_NAME}"
-}
-
-start_services() {
-  echo "Enabling and starting Elasticsearch service..."
-  sudo systemctl enable elasticsearch
-  sudo systemctl start elasticsearch
-  echo "Starting tixel-watch service..."
-  sudo systemctl start "${GO_SERVICE_NAME}"
-}
-
-main() {
-  if command -v apt-get &>/dev/null; then
-    install_es_deb
-  elif command -v yum &>/dev/null || command -v dnf &>/dev/null; then
-    install_es_rpm
-  else
-    echo "Unsupported package manager. Aborting."
-    exit 1
-  fi
-
-  setup_configuration
-  setup_tixel_watch_service
-  start_services
-  echo "All done."
-}
-
-main "$@"

local_storage.go

@@ -4,28 +4,277 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
-	"tixel_watch/models"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+	"watch-tool/models"
 
 	_ "modernc.org/sqlite"
 )
 
-type StorageService struct {
+type SQLiteStorage struct {
-	db *sql.DB
+	db           *sql.DB
+	dbPath       string
+	rotationCfg  StorageRotationConfig
+	rotationStop chan struct{}
+	rotationWg   sync.WaitGroup
+	mu           sync.RWMutex
 }
 
-func NewStorageService(dbPath string) (*StorageService, error) {
+func DefaultRotationConfig() StorageRotationConfig {
-	db, err := sql.Open("sqlite", dbPath)
+	return StorageRotationConfig{
+		MaxSizeBytes:         100 * 1024 * 1024, // 100MB
+		MaxAgeHours:          48 * time.Hour,    // 48 hours
+		MaxFiles:             3,                 // 3 old Files
+		CheckIntervalMinutes: 5 * time.Minute,   // check every 5 minutes
+		ArchiveDir:           "",                // same directory
+	}
+}
+
+func NewSQLiteStorage(dbPath string) (*SQLiteStorage, error) {
+	return NewSQLiteStorageWithRotation(dbPath, StorageRotationConfig{})
+}
+
+func NewSQLiteStorageWithRotation(dbPath string, rotationCfg StorageRotationConfig) (*SQLiteStorage, error) {
+	if rotationCfg.CheckIntervalMinutes == 0 {
+		rotationCfg = DefaultRotationConfig()
+	}
+	dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", dbPath)
+
+	db, err := sql.Open("sqlite", dsn)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to open SQLite database: %w", err)
+	}
+
+	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		return nil, fmt.Errorf("failed to enable WAL mode: %w", err)
+	}
+
+	if err := createTables(db); err != nil {
+		return nil, fmt.Errorf("failed to create tables: %w", err)
+	}
+
+	storage := &SQLiteStorage{
+		db:           db,
+		dbPath:       dbPath,
+		rotationCfg:  rotationCfg,
+		rotationStop: make(chan struct{}),
+	}
+
+	if rotationCfg.MaxSizeBytes > 0 || rotationCfg.MaxAgeHours > 0 {
+		storage.rotationWg.Add(1)
+		go storage.rotationWorker()
+		slog.Info("Log rotation enabled",
+			"maxSize", rotationCfg.MaxSizeBytes,
+			"maxAge", rotationCfg.MaxAgeHours,
+			"maxFiles", rotationCfg.MaxFiles)
+	}
+
+	return storage, nil
+}
+
+func (s *SQLiteStorage) rotationWorker() {
+	defer s.rotationWg.Done()
+	ticker := time.NewTicker(s.rotationCfg.CheckIntervalMinutes)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-s.rotationStop:
+			return
+		case <-ticker.C:
+			if err := s.checkAndRotate(); err != nil {
+				slog.Error("Error during log rotation check", "error", err)
+			}
+		}
+	}
+}
+
+func (s *SQLiteStorage) checkAndRotate() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	needsRotation, reason, err := s.needsRotation()
+	if err != nil {
+		return fmt.Errorf("error checking rotation needs: %w", err)
+	}
+
+	if needsRotation {
+		slog.Info("Starting log rotation", "reason", reason)
+		if err := s.rotateDatabase(); err != nil {
+			return fmt.Errorf("error rotating database: %w", err)
+		}
+		slog.Info("Log rotation completed successfully")
+	}
+
+	return nil
+}
+
+func (s *SQLiteStorage) needsRotation() (bool, string, error) {
+	if s.rotationCfg.MaxSizeBytes > 0 {
+		fileInfo, err := os.Stat(s.dbPath)
+		if err != nil {
+			return false, "", err
+		}
+		if fileInfo.Size() >= s.rotationCfg.MaxSizeBytes {
+			return true, fmt.Sprintf("file size %d >= max size %d", fileInfo.Size(), s.rotationCfg.MaxSizeBytes), nil
+		}
+	}
+
+	if s.rotationCfg.MaxAgeHours > 0 {
+		fileInfo, err := os.Stat(s.dbPath)
+		if err != nil {
+			return false, "", err
+		}
+		age := time.Since(fileInfo.ModTime())
+		if age >= s.rotationCfg.MaxAgeHours {
+			return true, fmt.Sprintf("file age %v >= max age %v", age, s.rotationCfg.MaxAgeHours), nil
+		}
+	}
+
+	return false, "", nil
+}
+
+func (s *SQLiteStorage) rotateDatabase() error {
+	if err := s.db.Close(); err != nil {
+		return fmt.Errorf("error closing database: %w", err)
+	}
+
+	archivePath := s.generateArchivePath()
+
+	if err := os.Rename(s.dbPath, archivePath); err != nil {
+		return fmt.Errorf("error moving database to archive: %w", err)
+	}
+
+	db, err := sql.Open("sqlite", s.dbPath)
+	if err != nil {
+		return fmt.Errorf("error opening new database: %w", err)
+	}
+
+	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		return fmt.Errorf("failed to enable WAL mode on new database: %w", err)
+	}
+
+	if err := createTables(db); err != nil {
+		return fmt.Errorf("failed to create tables in new database: %w", err)
+	}
+
+	s.db = db
+
+	if err := s.cleanupOldArchives(); err != nil {
+		slog.Warn("Error cleaning up old archives", "error", err)
+	}
+
+	return nil
+}
+
+func (s *SQLiteStorage) generateArchivePath() string {
+	dir := filepath.Dir(s.dbPath)
+	if s.rotationCfg.ArchiveDir != "" {
+		dir = s.rotationCfg.ArchiveDir
+		os.MkdirAll(dir, 0755)
+	}
+
+	base := filepath.Base(s.dbPath)
+	ext := filepath.Ext(base)
+	name := strings.TrimSuffix(base, ext)
+
+	timestamp := time.Now().Format("2006-01-02_15-04-05")
+	archiveName := fmt.Sprintf("%s.%s%s", name, timestamp, ext)
+
+	return filepath.Join(dir, archiveName)
+}
+
+func (s *SQLiteStorage) cleanupOldArchives() error {
+	if s.rotationCfg.MaxFiles <= 0 {
+		return nil
+	}
+
+	dir := filepath.Dir(s.dbPath)
+	if s.rotationCfg.ArchiveDir != "" {
+		dir = s.rotationCfg.ArchiveDir
+	}
+
+	base := filepath.Base(s.dbPath)
+	ext := filepath.Ext(base)
+	name := strings.TrimSuffix(base, ext)
+	pattern := fmt.Sprintf("%s.*%s", name, ext)
+
+	files, err := filepath.Glob(filepath.Join(dir, pattern))
+	if err != nil {
+		return err
+	}
+
+	var archives []string
+	for _, file := range files {
+		if file != s.dbPath {
+			archives = append(archives, file)
+		}
+	}
+
+	sort.Slice(archives, func(i, j int) bool {
+		infoI, _ := os.Stat(archives[i])
+		infoJ, _ := os.Stat(archives[j])
+		return infoI.ModTime().After(infoJ.ModTime())
+	})
+
+	if len(archives) > s.rotationCfg.MaxFiles {
+		for _, file := range archives[s.rotationCfg.MaxFiles:] {
+			if err := os.Remove(file); err != nil {
+				slog.Warn("Error removing old archive", "file", file, "error", err)
+			} else {
+				slog.Info("Removed old archive", "file", file)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *SQLiteStorage) ForceRotate() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	slog.Info("Forcing log rotation")
+	return s.rotateDatabase()
+}
+
+func (s *SQLiteStorage) GetRotationInfo() (map[string]any, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	fileInfo, err := os.Stat(s.dbPath)
 	if err != nil {
 		return nil, err
 	}
 
+	info := map[string]any{
+		"currentSize":   fileInfo.Size(),
+		"maxSize":       s.rotationCfg.MaxSizeBytes,
+		"currentAge":    time.Since(fileInfo.ModTime()).String(),
+		"maxAge":        s.rotationCfg.MaxAgeHours.String(),
+		"maxFiles":      s.rotationCfg.MaxFiles,
+		"checkInterval": s.rotationCfg.CheckIntervalMinutes.String(),
+		"archiveDir":    s.rotationCfg.ArchiveDir,
+	}
+
+	return info, nil
+}
+
+func createTables(db *sql.DB) error {
 	createTableStmt := `
 	CREATE TABLE IF NOT EXISTS log_entries (
 		id INTEGER PRIMARY KEY AUTOINCREMENT,
 		service TEXT,
-		timestamp DATETIME,
+		timestamp DATETIME NOT NULL,
-		type TEXT,
+		type TEXT NOT NULL,
-		host TEXT,
+		host TEXT NOT NULL,
 		tool TEXT,
 		log_level TEXT,
 		log_message TEXT,
@@ -39,143 +288,317 @@ func NewStorageService(dbPath string) (*StorageService, error) {
 		fields TEXT,
 		service_information TEXT,
 		system_metrics TEXT,
-		tool_information TEXT
+		tool_information TEXT,
-	);
+		created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-	`
+		exported_at DATETIME
-	_, err = db.ExecContext(context.Background(), createTableStmt)
+	);`
-	if err != nil {
+
-		return nil, err
+	if _, err := db.Exec(createTableStmt); err != nil {
+		return err
 	}
 
-	return &StorageService{db: db}, nil
+	indexes := []string{
+		"CREATE INDEX IF NOT EXISTS idx_timestamp ON log_entries(timestamp);",
+		"CREATE INDEX IF NOT EXISTS idx_service ON log_entries(service);",
+		"CREATE INDEX IF NOT EXISTS idx_type ON log_entries(type);",
+		"CREATE INDEX IF NOT EXISTS idx_tool ON log_entries(tool);",
+		"CREATE INDEX IF NOT EXISTS idx_log_level ON log_entries(log_level);",
+		"CREATE INDEX IF NOT EXISTS idx_exported ON log_entries(exported_at);",
+		"CREATE INDEX IF NOT EXISTS idx_composite ON log_entries(timestamp, type, service);",
+	}
+
+	for _, index := range indexes {
+		if _, err := db.Exec(index); err != nil {
+			return fmt.Errorf("failed to create index: %w", err)
+		}
+	}
+
+	return nil
 }
 
-func (s *StorageService) Close() error {
+func (s *SQLiteStorage) Store(ctx context.Context, entry *models.LogMessage) error {
+	return s.StoreBatch(ctx, []models.LogMessage{*entry})
+}
+
+func (s *SQLiteStorage) StoreBatch(ctx context.Context, entries []models.LogMessage) error {
+	if len(entries) == 0 {
+		return nil
+	}
+
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("failed to begin transaction: %w", err)
+	}
+	defer tx.Rollback()
+
+	stmt, err := tx.PrepareContext(ctx, `
+		INSERT INTO log_entries 
+		(service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, 
+		 unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	if err != nil {
+		return fmt.Errorf("failed to prepare statement: %w", err)
+	}
+	defer stmt.Close()
+
+	for _, entry := range entries {
+		fieldsJSON, _ := json.Marshal(entry.Fields)
+		serviceInfoJSON, _ := json.Marshal(entry.ServiceInformation)
+		systemMetricsJSON, _ := json.Marshal(entry.SystemMetrics)
+		toolInfoJSON, _ := json.Marshal(entry.ToolInformation)
+
+		_, err := stmt.ExecContext(ctx,
+			entry.Service,
+			entry.Timestamp,
+			entry.Type,
+			entry.Host,
+			entry.Tool,
+			entry.LogLevel,
+			entry.LogMessage,
+			entry.Raw,
+			entry.Priority,
+			entry.PriorityName,
+			entry.Unit,
+			entry.PID,
+			entry.BootID,
+			entry.MachineID,
+			string(fieldsJSON),
+			string(serviceInfoJSON),
+			string(systemMetricsJSON),
+			string(toolInfoJSON),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to insert entry: %w", err)
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (s *SQLiteStorage) Query(ctx context.Context, query StorageQuery) ([]models.LogMessage, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	sqlQuery := "SELECT service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information FROM log_entries WHERE 1=1"
+	args := []any{}
+	argCount := 0
+
+	if !query.StartTime.IsZero() {
+		argCount++
+		sqlQuery += fmt.Sprintf(" AND timestamp >= ?%d", argCount)
+		args = append(args, query.StartTime)
+	}
+	if !query.EndTime.IsZero() {
+		argCount++
+		sqlQuery += fmt.Sprintf(" AND timestamp <= ?%d", argCount)
+		args = append(args, query.EndTime)
+	}
+	if query.Service != "" {
+		argCount++
+		sqlQuery += fmt.Sprintf(" AND service = ?%d", argCount)
+		args = append(args, query.Service)
+	}
+	if query.Tool != "" {
+		argCount++
+		sqlQuery += fmt.Sprintf(" AND tool = ?%d", argCount)
+		args = append(args, query.Tool)
+	}
+	if query.LogLevel != "" {
+		argCount++
+		sqlQuery += fmt.Sprintf(" AND log_level = ?%d", argCount)
+		args = append(args, query.LogLevel)
+	}
+	if query.Type != "" {
+		argCount++
+		sqlQuery += fmt.Sprintf(" AND type = ?%d", argCount)
+		args = append(args, query.Type)
+	}
+
+	if query.OrderBy != "" {
+		direction := "ASC"
+		if query.OrderDesc {
+			direction = "DESC"
+		}
+		sqlQuery += fmt.Sprintf(" ORDER BY %s %s", query.OrderBy, direction)
+	} else {
+		sqlQuery += " ORDER BY timestamp DESC"
+	}
+
+	if query.Limit > 0 {
+		sqlQuery += fmt.Sprintf(" LIMIT %d", query.Limit)
+		if query.Offset > 0 {
+			sqlQuery += fmt.Sprintf(" OFFSET %d", query.Offset)
+		}
+	}
+
+	rows, err := s.db.QueryContext(ctx, sqlQuery, args...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query: %w", err)
+	}
+	defer rows.Close()
+
+	var entries []models.LogMessage
+	for rows.Next() {
+		var entry models.LogMessage
+		var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string
+
+		err := rows.Scan(
+			&entry.Service,
+			&entry.Timestamp,
+			&entry.Type,
+			&entry.Host,
+			&entry.Tool,
+			&entry.LogLevel,
+			&entry.LogMessage,
+			&entry.Raw,
+			&entry.Priority,
+			&entry.PriorityName,
+			&entry.Unit,
+			&entry.PID,
+			&entry.BootID,
+			&entry.MachineID,
+			&fieldsJSON,
+			&serviceInfoJSON,
+			&systemMetricsJSON,
+			&toolInfoJSON,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+
+		if fieldsJSON != "" && fieldsJSON != "null" {
+			json.Unmarshal([]byte(fieldsJSON), &entry.Fields)
+		}
+		if serviceInfoJSON != "" && serviceInfoJSON != "null" {
+			json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation)
+		}
+		if systemMetricsJSON != "" && systemMetricsJSON != "null" {
+			json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics)
+		}
+		if toolInfoJSON != "" && toolInfoJSON != "null" {
+			json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation)
+		}
+
+		entries = append(entries, entry)
+	}
+
+	return entries, rows.Err()
+}
+
+func (s *SQLiteStorage) MarkAsExported(ctx context.Context, ids []int64) error {
+	if len(ids) == 0 {
+		return nil
+	}
+
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	placeholders := strings.Repeat("?,", len(ids))
+	placeholders = placeholders[:len(placeholders)-1]
+
+	sqlQuery := fmt.Sprintf("UPDATE log_entries SET exported_at = CURRENT_TIMESTAMP WHERE id IN (%s)", placeholders)
+
+	args := make([]any, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	_, err = tx.ExecContext(ctx, sqlQuery, args...)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (s *SQLiteStorage) GetUnexportedEntries(ctx context.Context, limit int) ([]models.LogMessage, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	sqlQuery := `SELECT id, service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, 
+				unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information 
+				FROM log_entries WHERE exported_at IS NULL ORDER BY timestamp ASC`
+
+	if limit > 0 {
+		sqlQuery += fmt.Sprintf(" LIMIT %d", limit)
+	}
+
+	rows, err := s.db.QueryContext(ctx, sqlQuery)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query: %w", err)
+	}
+	defer rows.Close()
+
+	var entries []models.LogMessage
+	for rows.Next() {
+		var entry models.LogMessage
+		var id int64
+		var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string
+
+		err := rows.Scan(
+			&id,
+			&entry.Service,
+			&entry.Timestamp,
+			&entry.Type,
+			&entry.Host,
+			&entry.Tool,
+			&entry.LogLevel,
+			&entry.LogMessage,
+			&entry.Raw,
+			&entry.Priority,
+			&entry.PriorityName,
+			&entry.Unit,
+			&entry.PID,
+			&entry.BootID,
+			&entry.MachineID,
+			&fieldsJSON,
+			&serviceInfoJSON,
+			&systemMetricsJSON,
+			&toolInfoJSON,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+
+		if entry.Fields == nil {
+			entry.Fields = make(map[string]any)
+		}
+		entry.Fields["_internal_id"] = id
+
+		if fieldsJSON != "" && fieldsJSON != "null" {
+			json.Unmarshal([]byte(fieldsJSON), &entry.Fields)
+		}
+		if serviceInfoJSON != "" && serviceInfoJSON != "null" {
+			json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation)
+		}
+		if systemMetricsJSON != "" && systemMetricsJSON != "null" {
+			json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics)
+		}
+		if toolInfoJSON != "" && toolInfoJSON != "null" {
+			json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation)
+		}
+
+		entries = append(entries, entry)
+	}
+
+	return entries, rows.Err()
+}
+
+func (s *SQLiteStorage) Close() error {
+	close(s.rotationStop)
+	s.rotationWg.Wait()
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
 	return s.db.Close()
 }
-
-func (s *StorageService) SaveLogEntry(ctx context.Context, entry *models.LogMessage) error {
-	fieldsJSON := ""
-	if entry.Fields != nil {
-		b, err := json.Marshal(entry.Fields)
-		if err != nil {
-			return err
-		}
-		fieldsJSON = string(b)
-	}
-
-	serviceInfoJSON := ""
-	if entry.ServiceInformation != nil {
-		b, err := json.Marshal(entry.ServiceInformation)
-		if err != nil {
-			return err
-		}
-		serviceInfoJSON = string(b)
-	}
-
-	systemMetricsJSON := ""
-	if entry.SystemMetrics != nil {
-		b, err := json.Marshal(entry.SystemMetrics)
-		if err != nil {
-			return err
-		}
-		systemMetricsJSON = string(b)
-	}
-
-	toolInfoJSON := ""
-	if entry.ToolInformation != nil {
-		b, err := json.Marshal(entry.ToolInformation)
-		if err != nil {
-			return err
-		}
-		toolInfoJSON = string(b)
-	}
-
-	stmt := `
-	INSERT INTO log_entries 
-	(service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information)
-	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-	`
-	_, err := s.db.ExecContext(ctx, stmt,
-		entry.Service,
-		entry.Timestamp,
-		entry.Type,
-		entry.Host,
-		entry.Tool,
-		entry.LogLevel,
-		entry.LogMessage,
-		entry.Raw,
-		entry.Priority,
-		entry.PriorityName,
-		entry.Unit,
-		entry.PID,
-		entry.BootID,
-		entry.MachineID,
-		fieldsJSON,
-		serviceInfoJSON,
-		systemMetricsJSON,
-		toolInfoJSON,
-	)
-	return err
-}
-
-func (s *StorageService) LoadLogEntry(ctx context.Context, id int64) (*models.LogMessage, error) {
-	row := s.db.QueryRowContext(ctx, "SELECT service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information FROM log_entries WHERE id = ?", id)
-
-	var entry models.LogMessage
-	var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string
-
-	err := row.Scan(
-		&entry.Service,
-		&entry.Timestamp,
-		&entry.Type,
-		&entry.Host,
-		&entry.Tool,
-		&entry.LogLevel,
-		&entry.LogMessage,
-		&entry.Raw,
-		&entry.Priority,
-		&entry.PriorityName,
-		&entry.Unit,
-		&entry.PID,
-		&entry.BootID,
-		&entry.MachineID,
-		&fieldsJSON,
-		&serviceInfoJSON,
-		&systemMetricsJSON,
-		&toolInfoJSON,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	if fieldsJSON != "" {
-		var fields map[string]any
-		if err = json.Unmarshal([]byte(fieldsJSON), &fields); err == nil {
-			entry.Fields = fields
-		}
-	}
-
-	if serviceInfoJSON != "" {
-		var si any
-		if err = json.Unmarshal([]byte(serviceInfoJSON), &si); err == nil {
-			entry.ServiceInformation = si
-		}
-	}
-
-	if systemMetricsJSON != "" {
-		var sm any
-		if err = json.Unmarshal([]byte(systemMetricsJSON), &sm); err == nil {
-			entry.SystemMetrics = sm
-		}
-	}
-
-	if toolInfoJSON != "" {
-		var ti any
-		if err = json.Unmarshal([]byte(toolInfoJSON), &ti); err == nil {
-			entry.ToolInformation = ti
-		}
-	}
-
-	return &entry, nil
-}

local_storageV2.go

@@ -1,356 +0,0 @@
-package main
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-	"strings"
-	"tixel_watch/models"
-
-	_ "modernc.org/sqlite"
-)
-
-type SQLiteStorage struct {
-	db *sql.DB
-}
-
-func NewSQLiteStorage(dbPath string) (*SQLiteStorage, error) {
-	db, err := sql.Open("sqlite", dbPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open SQLite database: %w", err)
-	}
-
-	if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil {
-		return nil, fmt.Errorf("failed to enable WAL mode: %w", err)
-	}
-
-	if err := createTables(db); err != nil {
-		return nil, fmt.Errorf("failed to create tables: %w", err)
-	}
-
-	return &SQLiteStorage{db: db}, nil
-}
-
-func createTables(db *sql.DB) error {
-	createTableStmt := `
-	CREATE TABLE IF NOT EXISTS log_entries (
-		id INTEGER PRIMARY KEY AUTOINCREMENT,
-		service TEXT,
-		timestamp DATETIME NOT NULL,
-		type TEXT NOT NULL,
-		host TEXT NOT NULL,
-		tool TEXT,
-		log_level TEXT,
-		log_message TEXT,
-		raw TEXT,
-		priority TEXT,
-		priority_name TEXT,
-		unit TEXT,
-		pid INTEGER,
-		boot_id TEXT,
-		machine_id TEXT,
-		fields TEXT,
-		service_information TEXT,
-		system_metrics TEXT,
-		tool_information TEXT,
-		created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-		exported_at DATETIME
-	);`
-
-	if _, err := db.Exec(createTableStmt); err != nil {
-		return err
-	}
-
-	indexes := []string{
-		"CREATE INDEX IF NOT EXISTS idx_timestamp ON log_entries(timestamp);",
-		"CREATE INDEX IF NOT EXISTS idx_service ON log_entries(service);",
-		"CREATE INDEX IF NOT EXISTS idx_type ON log_entries(type);",
-		"CREATE INDEX IF NOT EXISTS idx_tool ON log_entries(tool);",
-		"CREATE INDEX IF NOT EXISTS idx_log_level ON log_entries(log_level);",
-		"CREATE INDEX IF NOT EXISTS idx_exported ON log_entries(exported_at);",
-		"CREATE INDEX IF NOT EXISTS idx_composite ON log_entries(timestamp, type, service);",
-	}
-
-	for _, index := range indexes {
-		if _, err := db.Exec(index); err != nil {
-			return fmt.Errorf("failed to create index: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (s *SQLiteStorage) Store(ctx context.Context, entry *models.LogMessage) error {
-	return s.StoreBatch(ctx, []models.LogMessage{*entry})
-}
-
-func (s *SQLiteStorage) StoreBatch(ctx context.Context, entries []models.LogMessage) error {
-	if len(entries) == 0 {
-		return nil
-	}
-
-	tx, err := s.db.BeginTx(ctx, nil)
-	if err != nil {
-		return fmt.Errorf("failed to begin transaction: %w", err)
-	}
-	defer tx.Rollback()
-
-	stmt, err := tx.PrepareContext(ctx, `
-		INSERT INTO log_entries 
-		(service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, 
-		 unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-	if err != nil {
-		return fmt.Errorf("failed to prepare statement: %w", err)
-	}
-	defer stmt.Close()
-
-	for _, entry := range entries {
-		fieldsJSON, _ := json.Marshal(entry.Fields)
-		serviceInfoJSON, _ := json.Marshal(entry.ServiceInformation)
-		systemMetricsJSON, _ := json.Marshal(entry.SystemMetrics)
-		toolInfoJSON, _ := json.Marshal(entry.ToolInformation)
-
-		_, err := stmt.ExecContext(ctx,
-			entry.Service,
-			entry.Timestamp,
-			entry.Type,
-			entry.Host,
-			entry.Tool,
-			entry.LogLevel,
-			entry.LogMessage,
-			entry.Raw,
-			entry.Priority,
-			entry.PriorityName,
-			entry.Unit,
-			entry.PID,
-			entry.BootID,
-			entry.MachineID,
-			string(fieldsJSON),
-			string(serviceInfoJSON),
-			string(systemMetricsJSON),
-			string(toolInfoJSON),
-		)
-		if err != nil {
-			return fmt.Errorf("failed to insert entry: %w", err)
-		}
-	}
-
-	return tx.Commit()
-}
-
-func (s *SQLiteStorage) Query(ctx context.Context, query StorageQuery) ([]models.LogMessage, error) {
-	sqlQuery := "SELECT service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information FROM log_entries WHERE 1=1"
-	args := []any{}
-	argCount := 0
-
-	if !query.StartTime.IsZero() {
-		argCount++
-		sqlQuery += fmt.Sprintf(" AND timestamp >= ?%d", argCount)
-		args = append(args, query.StartTime)
-	}
-	if !query.EndTime.IsZero() {
-		argCount++
-		sqlQuery += fmt.Sprintf(" AND timestamp <= ?%d", argCount)
-		args = append(args, query.EndTime)
-	}
-	if query.Service != "" {
-		argCount++
-		sqlQuery += fmt.Sprintf(" AND service = ?%d", argCount)
-		args = append(args, query.Service)
-	}
-	if query.Tool != "" {
-		argCount++
-		sqlQuery += fmt.Sprintf(" AND tool = ?%d", argCount)
-		args = append(args, query.Tool)
-	}
-	if query.LogLevel != "" {
-		argCount++
-		sqlQuery += fmt.Sprintf(" AND log_level = ?%d", argCount)
-		args = append(args, query.LogLevel)
-	}
-	if query.Type != "" {
-		argCount++
-		sqlQuery += fmt.Sprintf(" AND type = ?%d", argCount)
-		args = append(args, query.Type)
-	}
-
-	if query.OrderBy != "" {
-		direction := "ASC"
-		if query.OrderDesc {
-			direction = "DESC"
-		}
-		sqlQuery += fmt.Sprintf(" ORDER BY %s %s", query.OrderBy, direction)
-	} else {
-		sqlQuery += " ORDER BY timestamp DESC"
-	}
-
-	if query.Limit > 0 {
-		sqlQuery += fmt.Sprintf(" LIMIT %d", query.Limit)
-		if query.Offset > 0 {
-			sqlQuery += fmt.Sprintf(" OFFSET %d", query.Offset)
-		}
-	}
-
-	rows, err := s.db.QueryContext(ctx, sqlQuery, args...)
-	if err != nil {
-		return nil, fmt.Errorf("failed to execute query: %w", err)
-	}
-	defer rows.Close()
-
-	var entries []models.LogMessage
-	for rows.Next() {
-		var entry models.LogMessage
-		var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string
-
-		err := rows.Scan(
-			&entry.Service,
-			&entry.Timestamp,
-			&entry.Type,
-			&entry.Host,
-			&entry.Tool,
-			&entry.LogLevel,
-			&entry.LogMessage,
-			&entry.Raw,
-			&entry.Priority,
-			&entry.PriorityName,
-			&entry.Unit,
-			&entry.PID,
-			&entry.BootID,
-			&entry.MachineID,
-			&fieldsJSON,
-			&serviceInfoJSON,
-			&systemMetricsJSON,
-			&toolInfoJSON,
-		)
-		if err != nil {
-			return nil, fmt.Errorf("failed to scan row: %w", err)
-		}
-
-		if fieldsJSON != "" && fieldsJSON != "null" {
-			json.Unmarshal([]byte(fieldsJSON), &entry.Fields)
-		}
-		if serviceInfoJSON != "" && serviceInfoJSON != "null" {
-			json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation)
-		}
-		if systemMetricsJSON != "" && systemMetricsJSON != "null" {
-			json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics)
-		}
-		if toolInfoJSON != "" && toolInfoJSON != "null" {
-			json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation)
-		}
-
-		entries = append(entries, entry)
-	}
-
-	return entries, rows.Err()
-}
-
-func (s *SQLiteStorage) MarkAsExported(ctx context.Context, ids []int64) error {
-	if len(ids) == 0 {
-		return nil
-	}
-
-	tx, err := s.db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-
-	placeholders := strings.Repeat("?,", len(ids))
-	placeholders = placeholders[:len(placeholders)-1]
-
-	sqlQuery := fmt.Sprintf("UPDATE log_entries SET exported_at = CURRENT_TIMESTAMP WHERE id IN (%s)", placeholders)
-
-	args := make([]any, len(ids))
-	for i, id := range ids {
-		args[i] = id
-	}
-
-	_, err = tx.ExecContext(ctx, sqlQuery, args...)
-	if err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	err = tx.Commit()
-	if err != nil {
-		return err
-	}
-
-	return err
-}
-
-func (s *SQLiteStorage) GetUnexportedEntries(ctx context.Context, limit int) ([]models.LogMessage, error) {
-	sqlQuery := `SELECT id, service, timestamp, type, host, tool, log_level, log_message, raw, priority, priority_name, 
-				unit, pid, boot_id, machine_id, fields, service_information, system_metrics, tool_information 
-				FROM log_entries WHERE exported_at IS NULL ORDER BY timestamp ASC`
-
-	if limit > 0 {
-		sqlQuery += fmt.Sprintf(" LIMIT %d", limit)
-	}
-
-	rows, err := s.db.QueryContext(ctx, sqlQuery)
-	if err != nil {
-		return nil, fmt.Errorf("failed to execute query: %w", err)
-	}
-	defer rows.Close()
-
-	var entries []models.LogMessage
-	for rows.Next() {
-		var entry models.LogMessage
-		var id int64
-		var fieldsJSON, serviceInfoJSON, systemMetricsJSON, toolInfoJSON string
-
-		err := rows.Scan(
-			&id,
-			&entry.Service,
-			&entry.Timestamp,
-			&entry.Type,
-			&entry.Host,
-			&entry.Tool,
-			&entry.LogLevel,
-			&entry.LogMessage,
-			&entry.Raw,
-			&entry.Priority,
-			&entry.PriorityName,
-			&entry.Unit,
-			&entry.PID,
-			&entry.BootID,
-			&entry.MachineID,
-			&fieldsJSON,
-			&serviceInfoJSON,
-			&systemMetricsJSON,
-			&toolInfoJSON,
-		)
-		if err != nil {
-			return nil, fmt.Errorf("failed to scan row: %w", err)
-		}
-
-		if entry.Fields == nil {
-			entry.Fields = make(map[string]any)
-		}
-		entry.Fields["_internal_id"] = id
-
-		if fieldsJSON != "" && fieldsJSON != "null" {
-			json.Unmarshal([]byte(fieldsJSON), &entry.Fields)
-		}
-		if serviceInfoJSON != "" && serviceInfoJSON != "null" {
-			json.Unmarshal([]byte(serviceInfoJSON), &entry.ServiceInformation)
-		}
-		if systemMetricsJSON != "" && systemMetricsJSON != "null" {
-			json.Unmarshal([]byte(systemMetricsJSON), &entry.SystemMetrics)
-		}
-		if toolInfoJSON != "" && toolInfoJSON != "null" {
-			json.Unmarshal([]byte(toolInfoJSON), &entry.ToolInformation)
-		}
-
-		entries = append(entries, entry)
-	}
-
-	return entries, rows.Err()
-}
-
-func (s *SQLiteStorage) Close() error {
-	return s.db.Close()
-}

log_processor.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"log/slog"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 type LogProcessor struct {

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"codeberg.org/pata1704/drain3"
 	"context"
 	"log/slog"
 	"os"
@@ -8,30 +9,60 @@ import (
 	"sync"
 	"syscall"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/helpers"
+	"watch-tool/models"
+	"watch-tool/patterns"
 )
 
-var hostname string
+var currentHostname string
 
 func init() {
 	var err error
-	hostname, err = os.Hostname()
+	currentHostname, err = os.Hostname()
 	if err != nil {
-		hostname = "unknown"
+		currentHostname = "unknown"
+		slog.Warn("Could not determine hostname, using fallback", "fallback", currentHostname)
 	}
 }
 
 func main() {
-	cfg, err := LoadConfigV2()
+	cfg, err := LoadConfig()
 	if err != nil {
-		slog.Error("error loading configuration", "error", err)
+		slog.Error("Startup failed: configuration error", "error", err)
 		os.Exit(1)
 	}
-	slog.Info("TIXEL System Monitor started")
+
+	slog.Info("System Monitor started", "hostname", currentHostname)
+
+	if err := patterns.GetInstance().Load(cfg.PatternsFile); err != nil {
+		slog.Error("Startup failed: could not load patterns", "file", cfg.PatternsFile, "error", err)
+		os.Exit(1)
+	}
+	slog.Info("Regex patterns loaded successfully", "file", cfg.PatternsFile)
+
+	var d3Cfg *drain3.Config
+	if cfg.Drain3.Enabled {
+		d3Cfg = &drain3.Config{
+			Depth:       cfg.Drain3.Depth,
+			SimTh:       cfg.Drain3.SimThreshold,
+			MaxChildren: cfg.Drain3.MaxChildren,
+		}
+		slog.Info("Drain3 anomaly detection enabled", "state_dir", cfg.Drain3.StateDir)
+	} else {
+		slog.Info("Drain3 anomaly detection disabled")
+	}
 
 	var storage StorageInterface
 	if cfg.LocalStorage.Enable {
-		sqliteStorage, err := NewSQLiteStorage(cfg.LocalStorage.DBPath)
+		rotationConfig := StorageRotationConfig{
+			MaxSizeBytes:         cfg.LocalStorage.RotationConfig.MaxSizeBytes,
+			MaxAgeHours:          cfg.LocalStorage.RotationConfig.GetMaxAge(),
+			MaxFiles:             cfg.LocalStorage.RotationConfig.MaxFiles,
+			CheckIntervalMinutes: cfg.LocalStorage.RotationConfig.GetCheckInterval(),
+			ArchiveDir:           cfg.LocalStorage.RotationConfig.ArchiveDir,
+		}
+
+		sqliteStorage, err := NewSQLiteStorageWithRotation(cfg.LocalStorage.DBPath, rotationConfig)
 		if err != nil {
 			slog.Error("failed to initialize SQLite storage", "error", err)
 			os.Exit(1)
@@ -57,7 +88,7 @@ func main() {
 		exportManager = NewExportManager(storage, exportConfig)
 
 		if cfg.Elasticsearch.Enabled {
-			esExporter, err := NewElasticsearchExporterV2(cfg.Elasticsearch)
+			esExporter, err := NewElasticsearchExporter(cfg.Elasticsearch)
 			if err != nil {
 				slog.Error("failed to create Elasticsearch exporter", "error", err)
 				os.Exit(1)
@@ -71,10 +102,6 @@ func main() {
 			exportManager.RegisterExporter("elasticsearch", esExporter)
 			slog.Info("Elasticsearch exporter registered")
 		}
-
-		// Add more exporters here in the future
-		// exportManager.RegisterExporter("checkmk", checkmkExporter)
-		// exportManager.RegisterExporter("grafana", grafanaExporter)
 	}
 
 	logChan := make(chan models.LogMessage, 1000)
@@ -84,86 +111,92 @@ func main() {
 	var wg sync.WaitGroup
 
 	wg.Add(1)
-	go func() {
+	helpers.SafeGo(ctx, "LogProcessor", func() {
 		defer wg.Done()
 		processor := NewLogProcessor(storage)
 		processor.Start(ctx, logChan)
-	}()
+	})
 
 	if exportManager != nil {
 		wg.Add(1)
-		go func() {
+		helpers.SafeGo(ctx, "ExportManager", func() {
 			defer wg.Done()
 			exportManager.Start(ctx)
-		}()
+		})
 	}
 
 	for _, service := range cfg.Services {
 		if !service.Enabled {
-			slog.Info("Service deactivated, skipping...", "service", service.Name)
+			slog.Debug("Service deactivated, skipping...", "service", service.Name)
 			continue
 		}
 
 		wg.Add(1)
-		go func(s ServiceConfig) {
+		srv := service
-			defer wg.Done()
-			monitor := NewServiceMonitor(s)
-			if err := monitor.Start(ctx, logChan); err != nil {
-				slog.Error("error watching service", "service", s.Name, "error", err)
-			}
-		}(service)
 
-		slog.Info("started watching Service-Log", "service", service.Name)
+		helpers.SafeGo(ctx, "ServiceMonitor-"+srv.Name, func() {
+			defer wg.Done()
+			monitor := NewServiceMonitor(srv, currentHostname, d3Cfg, cfg.Drain3.StateDir)
+
+			if err := monitor.Start(ctx, logChan); err != nil {
+				slog.Error("Error watching service", "service", srv.Name, "error", err)
+			}
+		})
+
+		slog.Info("Started watching Service-Log", "service", service.Name)
 	}
 
 	for _, tool := range cfg.Tools {
 		if !tool.Enabled {
-			slog.Info("Tool is deactivated, skipping...", "tool", tool.Name)
+			slog.Debug("Tool is deactivated, skipping...", "tool", tool.Name)
 			continue
 		}
 
 		wg.Add(1)
-		go func(t ToolConfig) {
+		t := tool
-			defer wg.Done()
-			monitor := NewFileMonitor(t)
-			if err := monitor.Start(ctx, logChan); err != nil {
-				slog.Error("error watching", "tool", t.Name, "error", err)
-			}
-		}(tool)
 
-		slog.Info("started watching logs", "tool", tool.Name, "file", tool.LogFile)
+		helpers.SafeGo(ctx, "FileMonitor-"+t.Name, func() {
+			defer wg.Done()
+
+			monitor := NewFileMonitor(t, currentHostname, d3Cfg, cfg.Drain3.StateDir)
+
+			if err := monitor.Start(ctx, logChan); err != nil {
+				slog.Error("Error watching tool", "tool", t.Name, "error", err)
+			}
+		})
+
+		slog.Info("Started watching logs", "tool", tool.Name, "file", tool.LogFile)
 	}
 
 	if cfg.SystemMetrics.Enabled {
 		wg.Add(1)
-		go func() {
+		helpers.SafeGo(ctx, "SystemMetrics", func() {
 			defer wg.Done()
-			collector := NewSystemMetricsCollector(cfg.SystemMetrics, cfg.PollIntervalSeconds)
+			collector := NewSystemMetricsCollector(cfg.SystemMetrics, cfg.PollIntervalSeconds, currentHostname)
-			collector.StartV2(ctx, storage, logChan)
+			collector.Start(ctx, storage, logChan)
-		}()
+		})
 		slog.Info("Started collecting System-Metrics")
 	}
 
 	if cfg.WebService.Enabled {
 		wg.Add(1)
-		go func() {
+		helpers.SafeGo(ctx, "WebService", func() {
 			defer wg.Done()
-			webService := NewWebServiceV2(cfg, storage)
+			webService := NewWebService(cfg, storage)
 			if err := webService.Start(ctx); err != nil {
-				slog.Error("web service error", "error", err)
+				slog.Error("Web service error", "error", err)
 			}
-		}()
+		})
 		slog.Info("Web service started", "host", cfg.WebService.Host, "port", cfg.WebService.Port)
 	}
 
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
 
-	<-sigCh
+	s := <-sigCh
-	slog.Info("Shutdown-Signal received, stopping threads...")
+	slog.Info("Shutdown signal received, stopping threads...", "signal", s)
 
 	cancel()
-	close(logChan)
 
 	done := make(chan struct{})
 	go func() {
@@ -173,9 +206,11 @@ func main() {
 
 	select {
 	case <-done:
-		slog.Info("All threads closed")
+		close(logChan)
+		slog.Info("All threads closed gracefully")
 	case <-time.After(10 * time.Second):
-		slog.Info("Shutdown-Timeout reached, force quitting")
+		slog.Error("Shutdown timeout reached, force quitting")
+		os.Exit(1)
 	}
 
 	slog.Info("Program stopped")

models/models.go

@@ -136,18 +136,8 @@ type LogMessage struct {
 	BootID             string         `json:"boot_id,omitempty"`
 	MachineID          string         `json:"machine_id,omitempty"`
 	Fields             map[string]any `json:"fields,omitempty"`
-	// SyslogInfo         SyslogFields   `json:"syslog_information,omitempty"`
 }
 
-// type LogMessage struct {
-// 	Service            string       `json:"service"`
-// 	Timestamp          time.Time    `json:"timestamp"`
-// 	LogLevel           string       `json:"log_level"`
-// 	LogMessage         string       `json:"log_message"`
-// 	SyslogInfo         SyslogFields `json:"syslog_information"`
-// 	ServiceInformation any          `json:"service_info,omitempty"`
-// }
-
 type SyslogFields struct {
 	SysLogTimestamp time.Time `json:"syslog_timestamp"`
 	Hostname        string    `json:"hostname"`

parser/am_parser.go

@@ -1,49 +0,0 @@
-package parser
-
-import (
-	"log/slog"
-	"regexp"
-	"strings"
-	"time"
-	"tixel_watch/helpers"
-	"tixel_watch/models"
-)
-
-var (
-	amServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
-)
-
-type AMParser struct{}
-
-func (a *AMParser) Parse(line string) (models.LogMessage, error) {
-	newEntry := models.LogMessage{
-		Service: "access-manager",
-	}
-	syslogFields, logContent := helpers.ExtractSyslogHeader(line)
-	newEntry.Host = syslogFields.Hostname
-
-	matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
-	if len(matches) != 7 {
-		newEntry.Timestamp = time.Now()
-		newEntry.LogMessage = line
-		return newEntry, nil
-	}
-
-	timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
-	timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
-	if err != nil {
-		slog.Error("unable to parse time", "error", err)
-		return newEntry, err
-	}
-	baseInfo := models.AMBaseInfo{
-		ProcessID:  matches[3],
-		ThreadID:   strings.TrimSpace(matches[4]),
-		LoggerName: matches[5],
-	}
-	newEntry.Timestamp = timestamp
-	newEntry.LogLevel = matches[2]
-	newEntry.LogMessage = matches[6]
-	newEntry.ServiceInformation = baseInfo
-
-	return newEntry, nil
-}

parser/default_parser.go

@@ -1,28 +0,0 @@
-package parser
-
-import (
-	"strings"
-	"time"
-	"tixel_watch/models"
-)
-
-type DefaultParser struct {
-	Service string
-	Tool    string
-}
-
-func (d *DefaultParser) Parse(line string) (models.LogMessage, error) {
-	msg := models.LogMessage{
-		LogLevel:   "unknown",
-		LogMessage: strings.TrimSpace(line),
-		Raw:        line,
-		Timestamp:  time.Now(),
-	}
-	if d.Service != "" {
-		msg.Service = d.Service
-	}
-	if d.Tool != "" {
-		msg.Tool = d.Tool
-	}
-	return msg, nil
-}

parser/factory.go

@@ -1,27 +1,20 @@
 package parser
 
-func New(serviceName, logType string) (Parser, error) {
+import "codeberg.org/pata1704/drain3"
-	switch logType {
+
-	case "custom":
+type ParserConfig struct {
-		switch serviceName {
+	ServiceName string
-		case "tixstream":
+	LogType     string
-			return &TSParser{}, nil
+	Hostname    string
-		case "transfer-job-manager":
+	DrainConfig *drain3.Config
-			return &TJMParser{}, nil
+	StateDir    string
-		case "access-manager":
+}
-			return &AMParser{}, nil
+
-		case "tixel-control-center":
+func New(cfg ParserConfig) (Parser, error) {
-			return &TCCParser{}, nil
+	switch cfg.LogType {
-		case "nginx":
-			return &NginxParser{}, nil
-		case "nginx-tjm":
-			return &NginxTJMLogParser{ToolName: serviceName}, nil
-		default:
-			return &DefaultParser{Service: serviceName}, nil
-		}
 	case "json":
 		return &JSONParser{}, nil
 	default:
-		return &DefaultParser{Service: serviceName}, nil
+		return NewGenericParser(cfg.ServiceName, cfg.Hostname, cfg.DrainConfig, cfg.StateDir), nil
 	}
 }

parser/generic_parser.go

@@ -0,0 +1,231 @@
+package parser
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+	"watch-tool/models"
+	"watch-tool/patterns"
+
+	"codeberg.org/pata1704/drain3"
+)
+
+type GenericParser struct {
+	ServiceName    string
+	Hostname       string
+	Extractors     []patterns.CompiledExtractor
+	CommonExt      []patterns.CompiledExtractor
+	drainMiner     *drain3.TemplateMiner
+	drainEnabled   bool
+	drainStatePath string
+}
+
+func NewGenericParser(serviceName, hostname string, drainCfg *drain3.Config, stateDir string) *GenericParser {
+	repo := patterns.GetInstance()
+
+	var svcExt, commonExt []patterns.CompiledExtractor
+	if repo != nil {
+		svcExt = repo.GetExtractors(serviceName)
+		commonExt = repo.GetExtractors("common")
+	} else {
+		slog.Error("CRITICAL: Pattern Repository is nil. Parser will not work correctly.")
+	}
+
+	parser := &GenericParser{
+		ServiceName: serviceName,
+		Hostname:    hostname,
+		Extractors:  svcExt,
+		CommonExt:   commonExt,
+	}
+	if drainCfg != nil && stateDir != "" {
+		parser.drainEnabled = true
+
+		parser.drainStatePath = filepath.Join(stateDir, serviceName+".bin")
+
+		if err := os.MkdirAll(stateDir, 0755); err != nil {
+			slog.Error("Failed to create drain3 state dir", "error", err)
+			parser.drainEnabled = false
+			return parser
+		}
+
+		persister := drain3.NewFilePersistence(parser.drainStatePath, false)
+
+		state, err := persister.LoadState()
+		if err == nil && state != nil {
+			parser.drainMiner = drain3.NewTemplateMiner(drainCfg, persister)
+			slog.Info("Drain3 state loaded", "service", serviceName, "clusters", len(state.Clusters))
+		} else {
+			parser.drainMiner = drain3.NewTemplateMiner(drainCfg, persister)
+			slog.Info("Drain3 initialized fresh", "service", serviceName)
+		}
+	}
+
+	return parser
+}
+
+func (p *GenericParser) Parse(line string) (models.LogMessage, error) {
+	entry := models.LogMessage{
+		Service:   p.ServiceName,
+		Host:      p.Hostname,
+		Timestamp: time.Now(),
+		Raw:       line,
+		Fields:    make(map[string]any),
+		Type:      "log_entry",
+	}
+
+	trimmedLine := strings.TrimSpace(line)
+	if trimmedLine == "" {
+		return entry, nil
+	}
+
+	if p.drainEnabled && p.drainMiner != nil {
+		cluster := p.drainMiner.AddLogMessage(trimmedLine)
+		if cluster != nil {
+			entry.Fields["drain_template_id"] = cluster.ClusterID
+			entry.Fields["drain_template"] = cluster.TemplateMined
+			// Optional: Parameter extrahieren, die Drain gefunden hat (Wildcards)
+		}
+	}
+
+	allExtractors := append(p.CommonExt, p.Extractors...)
+
+	matchedAny := false
+
+	for _, ext := range allExtractors {
+		matches := ext.Pattern.FindStringSubmatch(trimmedLine)
+		if matches == nil {
+			continue
+		}
+		matchedAny = true
+
+		subexpNames := ext.Pattern.SubexpNames()
+		for i, matchValue := range matches {
+			if i == 0 {
+				continue
+			}
+
+			groupName := subexpNames[i]
+			if groupName == "" {
+				continue
+			}
+
+			cleanValue := strings.TrimSpace(matchValue)
+
+			targetType := ext.Fields[groupName]
+			parsedValue := p.safeConvert(cleanValue, targetType)
+
+			p.mapField(&entry, groupName, parsedValue)
+		}
+	}
+
+	if !matchedAny {
+		entry.LogMessage = trimmedLine
+		entry.Fields["_parse_status"] = "failed"
+	} else if entry.LogMessage == "" {
+		entry.LogMessage = trimmedLine
+	}
+
+	return entry, nil
+}
+
+func (p *GenericParser) Close() error {
+	if p.drainEnabled && p.drainMiner != nil {
+		if err := p.drainMiner.SaveState("shutdown"); err != nil {
+			slog.Error("Failed to save drain3 state", "service", p.ServiceName, "error", err)
+			return err
+		}
+		slog.Debug("Drain3 state saved", "service", p.ServiceName)
+	}
+	return nil
+}
+
+func (p *GenericParser) safeConvert(value, typeDef string) any {
+	if value == "" || value == "-" {
+		if strings.HasPrefix(typeDef, "int") || strings.HasPrefix(typeDef, "float") {
+			return 0
+		}
+		return value
+	}
+
+	var err error
+	var result any
+
+	switch {
+	case strings.HasPrefix(typeDef, "int"):
+		var i int
+		i, err = strconv.Atoi(value)
+		result = i
+
+	case strings.HasPrefix(typeDef, "float"):
+		var f float64
+		f, err = strconv.ParseFloat(value, 64)
+		result = f
+
+	case strings.HasPrefix(typeDef, "time:"):
+		layout := strings.TrimPrefix(typeDef, "time:")
+		result, err = p.parseTimeRobust(value, layout)
+
+	case typeDef == "bool":
+		var b bool
+		b, err = strconv.ParseBool(value)
+		result = b
+
+	default:
+		return value
+	}
+
+	if err != nil {
+		return value
+	}
+
+	return result
+}
+
+func (p *GenericParser) parseTimeRobust(value, layout string) (time.Time, error) {
+	if layout == "Jan 02 15:04:05" {
+		t, err := time.Parse(layout, value)
+		if err != nil {
+			return time.Time{}, err
+		}
+		now := time.Now()
+		year := now.Year()
+		if t.Month() > now.Month() {
+			year--
+		}
+		return t.AddDate(year, 0, 0), nil
+	}
+
+	return time.Parse(layout, value)
+}
+
+func (p *GenericParser) mapField(entry *models.LogMessage, key string, value any) {
+	switch key {
+	case "timestamp", "time":
+		if t, ok := value.(time.Time); ok {
+			entry.Timestamp = t
+		}
+	case "log_level", "level":
+		entry.LogLevel = fmt.Sprintf("%v", value)
+	case "message", "msg":
+		entry.LogMessage = fmt.Sprintf("%v", value)
+	case "host", "hostname":
+		entry.Host = fmt.Sprintf("%v", value)
+	case "service":
+		entry.Service = fmt.Sprintf("%v", value)
+	case "pid":
+		if v, ok := value.(int); ok {
+			entry.PID = v
+		} else if vStr, ok := value.(string); ok {
+			if pid, err := strconv.Atoi(vStr); err == nil {
+				entry.PID = pid
+			}
+		}
+
+	default:
+		entry.Fields[key] = value
+	}
+}

parser/generic_parser_test.go

@@ -0,0 +1,198 @@
+package parser
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+	"watch-tool/patterns"
+
+	"codeberg.org/pata1704/drain3"
+)
+
+func setupPatterns(t *testing.T) {
+	content := `
+patterns:
+  common:
+    extractors:
+      - name: "syslog_header"
+        regex: '^\w{3} \d{2} \d{2}:\d{2}:\d{2} (?P<hostname>\S+) .*'
+        fields:
+          hostname: "string"
+
+  test_service:
+    extractors:
+      - name: "data_line"
+        regex: 'Data: id=(?P<id>\d+) size=(?P<size_mb>[0-9.]+) active=(?P<is_active>true|false) empty=(?P<empty_val>\S+)'
+        fields:
+          id: "int"
+          size_mb: "float"
+          is_active: "bool"
+          empty_val: "int" # Testet Fallback bei "-"
+`
+	tmpfile, err := os.CreateTemp("", "patterns_parser_test_*.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(content)); err != nil {
+		t.Fatal(err)
+	}
+	tmpfile.Close()
+
+	if err := patterns.GetInstance().Load(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGenericParser_Parse_Regex(t *testing.T) {
+	setupPatterns(t)
+
+	p := NewGenericParser("test_service", "localhost", nil, "")
+
+	line := "Sep 28 10:00:00 myhost Data: id=42 size=12.5 active=true empty=-"
+	entry, err := p.Parse(line)
+	if err != nil {
+		t.Fatalf("Parse failed: %v", err)
+	}
+
+	if entry.Host != "myhost" {
+		t.Errorf("Expected host 'myhost', got '%s'", entry.Host)
+	}
+
+	if val, ok := entry.Fields["id"].(int); !ok || val != 42 {
+		t.Errorf("Expected id=42 (int), got %v (%T)", entry.Fields["id"], entry.Fields["id"])
+	}
+
+	if val, ok := entry.Fields["size_mb"].(float64); !ok || val != 12.5 {
+		t.Errorf("Expected size_mb=12.5 (float), got %v", entry.Fields["size_mb"])
+	}
+
+	if val, ok := entry.Fields["is_active"].(bool); !ok || val != true {
+		t.Errorf("Expected is_active=true, got %v", entry.Fields["is_active"])
+	}
+
+	if val, ok := entry.Fields["empty_val"].(int); !ok || val != 0 {
+		t.Errorf("Expected empty_val=0 for '-', got %v", entry.Fields["empty_val"])
+	}
+}
+
+func TestGenericParser_Drain3_Integration(t *testing.T) {
+	setupPatterns(t)
+
+	opts := &slog.HandlerOptions{Level: slog.LevelDebug}
+	logger := slog.New(slog.NewTextHandler(os.Stdout, opts))
+	slog.SetDefault(logger)
+
+	tmpDir, err := os.MkdirTemp("", "drain_state_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	drainCfg := &drain3.Config{
+		Depth:       4,
+		SimTh:       0.5,
+		MaxChildren: 100,
+		MaxClusters: 100,
+	}
+
+	serviceName := "test_service"
+	p := NewGenericParser(serviceName, "localhost", drainCfg, tmpDir)
+
+	log1 := "User admin logged in from 192.168.1.1"
+	log2 := "User guest logged in from 10.0.0.1"
+
+	entry1, _ := p.Parse(log1)
+	if entry1.Fields["drain_template_id"] == nil {
+		t.Error("Drain3 did not assign a template ID for log1")
+	}
+
+	entry2, _ := p.Parse(log2)
+
+	id1 := entry1.Fields["drain_template_id"]
+	id2 := entry2.Fields["drain_template_id"]
+	t.Logf("IDs: %v -> %v", id1, id2)
+	t.Logf("Template 2: %s", entry2.Fields["drain_template"])
+
+	if err := p.Close(); err != nil {
+		t.Fatalf("Close failed: %v", err)
+	}
+
+	expectedFile := filepath.Join(tmpDir, serviceName+".bin")
+
+	if info, err := os.Stat(expectedFile); os.IsNotExist(err) {
+		t.Errorf("Drain3 state file NOT found at: %s", expectedFile)
+
+		entries, _ := os.ReadDir(tmpDir)
+		t.Logf("Listing directory %s:", tmpDir)
+		for _, e := range entries {
+			t.Logf(" - Found file: %s", e.Name())
+		}
+	} else {
+		t.Logf("Success: State file found (%d bytes)", info.Size())
+	}
+}
+
+func TestGenericParser_Robustness(t *testing.T) {
+	setupPatterns(t)
+
+	p := NewGenericParser("test_service", "localhost", nil, "")
+
+	tests := []struct {
+		name          string
+		log           string
+		checkField    string
+		expectedValue any
+		shouldFail    bool
+	}{
+		{
+			name:          "Empty Line",
+			log:           "",
+			checkField:    "",
+			expectedValue: nil,
+			shouldFail:    false,
+		},
+		{
+			name:          "Type Mismatch Int (Text instead of Int)",
+			log:           "Data: id=abc size=12.5 active=true empty=-",
+			checkField:    "_parse_status",
+			expectedValue: "failed",
+		},
+		{
+			name:          "Value Missing (Dash) for Int",
+			log:           "Data: id=1 size=1.0 active=true empty=-",
+			checkField:    "empty_val",
+			expectedValue: 0,
+		},
+		{
+			name:          "Value Missing (Dash) for Float",
+			log:           "Data: id=1 size=1.0 active=true empty=0",
+			checkField:    "size_mb",
+			expectedValue: 1.0,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			entry, err := p.Parse(tc.log)
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if tc.checkField != "" {
+				val, exists := entry.Fields[tc.checkField]
+				if tc.expectedValue == "failed" {
+					if !exists || val != "failed" {
+						t.Errorf("Expected parse failure status, got %v", val)
+					}
+				} else {
+					if val != tc.expectedValue {
+						t.Errorf("Field %s: expected %v, got %v", tc.checkField, tc.expectedValue, val)
+					}
+				}
+			}
+		})
+	}
+}

parser/json_parser.go

@@ -3,7 +3,7 @@ package parser
 import (
 	"encoding/json"
 	"log/slog"
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 type JSONParser struct{}
@@ -17,3 +17,7 @@ func (j *JSONParser) Parse(line string) (models.LogMessage, error) {
 	}
 	return logMsg, nil
 }
+
+func (p *JSONParser) Close() error {
+	return nil
+}

parser/nginx_parser.go

@@ -1,57 +0,0 @@
-package parser
-
-import (
-	"log/slog"
-	"regexp"
-	"strconv"
-	"strings"
-	"tixel_watch/models"
-)
-
-var (
-	nginxAccessPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
-)
-
-type NginxParser struct{}
-
-func (n *NginxParser) Parse(line string) (models.LogMessage, error) {
-	newEntry := models.LogMessage{
-		Service: "nginx",
-	}
-
-	matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(line))
-	if len(matches) < 7 {
-		return newEntry, nil
-	}
-	statusCode, err := strconv.ParseInt(matches[5], 10, 64)
-	if err != nil {
-		slog.Error("cant parse statuscode", "error", err)
-	}
-	bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
-	if err != nil {
-		slog.Error("cant parse bytessend", "error", err)
-	}
-	baseInfo := models.NGinXBaseInfo{
-		ClientIP:   matches[1],
-		RemoteUser: matches[2],
-		Request:    matches[4],
-		StatusCode: int(statusCode),
-		BytesSend:  int(bytesSend),
-	}
-
-	if len(matches) > 7 && matches[7] != "" {
-		baseInfo.Referer = matches[7]
-	}
-	if len(matches) > 8 && matches[8] != "" {
-		baseInfo.UserAgent = matches[8]
-	}
-
-	if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
-		baseInfo.HTTPMethod = requestParts[0]
-		baseInfo.RequestURI = requestParts[1]
-		baseInfo.HTTPVersion = requestParts[2]
-	}
-	newEntry.ServiceInformation = baseInfo
-
-	return newEntry, nil
-}

parser/nginx_tjm_parser.go

@@ -4,12 +4,13 @@ import (
 	"log/slog"
 	"strconv"
 	"strings"
-	"tixel_watch/helpers"
+	"watch-tool/helpers"
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 type NginxTJMLogParser struct {
 	ToolName string
+	Hostname string
 }
 
 func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) {
@@ -18,11 +19,7 @@ func (p *NginxTJMLogParser) Parse(line string) (models.LogMessage, error) {
 		Tool: p.ToolName,
 		Raw:  line,
 	}
-	hostname, err := helpers.GetHostname()
+	entry.Host = p.Hostname
-	if err != nil {
-		return entry, err
-	}
-	entry.Host = hostname
 	entry = p.parseNginxTJM(entry)
 	return entry, nil
 }

parser/parser.go

@@ -1,11 +1,10 @@
 package parser
 
 import (
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 type Parser interface {
-	//TODO: Change parsers to return an error as well
 	Parse(line string) (models.LogMessage, error)
-	// Parse(line string) models.LogMessage
+	Close() error
 }

parser/regex_parser.go

@@ -1,64 +0,0 @@
-package parser
-
-import (
-	"log/slog"
-	"regexp"
-	"strings"
-	"tixel_watch/helpers"
-	"tixel_watch/models"
-)
-
-type RegexLogParser struct {
-	Pattern  *regexp.Regexp
-	Fields   map[string]string
-	Toolname string
-}
-
-func (p *RegexLogParser) Parse(line string) (models.LogMessage, error) {
-	entry := models.LogMessage{Type: "log_entry"}
-	entry.Tool = p.Toolname
-	entry.Raw = line
-	hostname, err := helpers.GetHostname()
-	if err != nil {
-		slog.Warn("cannot get hostname")
-		return entry, err
-	}
-	entry.Host = hostname
-
-	fields := p.parseWithPattern(line)
-	if fields != nil {
-		entry.Fields = fields
-	} else {
-		entry.LogMessage = strings.TrimSpace(line)
-	}
-
-	return entry, nil
-}
-
-func (p *RegexLogParser) parseWithPattern(text string) map[string]any {
-	matches := p.Pattern.FindStringSubmatch(text)
-	if matches == nil {
-		return nil
-	}
-
-	fields := make(map[string]any)
-	subexpNames := p.Pattern.SubexpNames()
-
-	for i, match := range matches {
-		if i == 0 {
-			continue
-		}
-
-		if i < len(subexpNames) && subexpNames[i] != "" {
-			fieldName := subexpNames[i]
-
-			if mappedName, exists := p.Fields[fieldName]; exists {
-				fieldName = mappedName
-			}
-
-			fields[fieldName] = match
-		}
-	}
-
-	return fields
-}

parser/tcc_parser.go

@@ -1,49 +0,0 @@
-package parser
-
-import (
-	"log/slog"
-	"regexp"
-	"strings"
-	"time"
-	"tixel_watch/helpers"
-	"tixel_watch/models"
-)
-
-var (
-	tccServicePattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
-)
-
-type TCCParser struct{}
-
-func (t *TCCParser) Parse(line string) (models.LogMessage, error) {
-	newEntry := models.LogMessage{
-		Service: "tixel-control-center",
-	}
-	syslogFields, logContent := helpers.ExtractSyslogHeader(line)
-	newEntry.Host = syslogFields.Hostname
-
-	matches := tccServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
-	if len(matches) != 7 {
-		newEntry.Timestamp = time.Now()
-		newEntry.LogMessage = line
-		return newEntry, nil
-	}
-
-	timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
-	timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
-	if err != nil {
-		slog.Error("unable to parse time", "error", err)
-		return newEntry, err
-	}
-	baseInfo := models.TCCBaseInfo{
-		ProcessID:  matches[3],
-		ThreadID:   strings.TrimSpace(matches[4]),
-		LoggerName: matches[5],
-	}
-	newEntry.Timestamp = timestamp
-	newEntry.LogLevel = matches[2]
-	newEntry.LogMessage = matches[6]
-	newEntry.ServiceInformation = baseInfo
-
-	return newEntry, nil
-}

parser/tjm_parser.go

@@ -1,91 +0,0 @@
-package parser
-
-import (
-	"log/slog"
-	"regexp"
-	"strings"
-	"tixel_watch/helpers"
-	"tixel_watch/models"
-)
-
-var (
-	tjmServicePattern      = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
-	tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (.*)$`)
-	tjmTransferIDPattern1  = regexp.MustCompile(`(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
-	tjmTransferIDPattern2  = regexp.MustCompile(`(?P<before>.*)(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
-)
-
-type TJMParser struct{}
-
-func (t *TJMParser) Parse(line string) (models.LogMessage, error) {
-	newEntry := models.LogMessage{
-		Service: "transfer-job-manager",
-	}
-	syslogFields, logContent := helpers.ExtractSyslogHeader(line)
-	newEntry.Host = syslogFields.Hostname
-
-	msg := strings.TrimSpace(logContent)
-	msg = strings.ReplaceAll(msg, "           ", " ")
-	msg = strings.ReplaceAll(msg, "---", "")
-	msg = strings.ReplaceAll(msg, "  ", " ")
-	parts := strings.Fields(msg)
-	if len(parts) < 4 {
-		newEntry.LogMessage = logContent
-		return newEntry, nil
-	}
-	matches := tjmServicePattern.FindStringSubmatch(logContent)
-	var baseInfo models.TJMTransferInfo
-	if len(matches) > 0 {
-		timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
-		timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
-		if err != nil {
-			slog.Error("unable to parse time", "error", err)
-		}
-		newEntry.Timestamp = timestamp
-		newEntry.LogLevel = strings.TrimSpace(matches[2])
-		newEntry.LogMessage = strings.TrimSpace(matches[8])
-		baseInfo = models.TJMTransferInfo{
-			ProcessID:     strings.TrimSpace(matches[3]),
-			CorrelationID: strings.TrimSpace(matches[4]),
-			Username:      strings.TrimSpace(matches[5]),
-			ThreadID:      strings.TrimSpace(matches[6]),
-			JavaClass:     strings.TrimSpace(matches[7]),
-		}
-	} else {
-		newEntry.LogMessage = logContent
-	}
-	trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
-	var transferName string
-	var transferID string
-	if len(trNameMatch) > 0 {
-		transferName = trNameMatch[1]
-		newEntry.LogMessage = trNameMatch[2]
-		if strings.Contains(trNameMatch[1], "-in") {
-			baseInfo.Direction = "incoming"
-		}
-		if strings.Contains(trNameMatch[1], "-out") {
-			baseInfo.Direction = "outgoing"
-		}
-	}
-	trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
-	if len(trIDMatch) > 0 {
-		transferID = trIDMatch[1]
-	}
-	trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
-	if len(trIDMatch) > 0 {
-		transferID = trIDMatch[2]
-	}
-	if transferID != "" {
-		baseInfo.TransferID = transferID
-	} else if transferName != "" {
-		baseInfo.TransferID = transferName
-	} else {
-		baseInfo.TransferID = "no_transfer_id"
-	}
-	if baseInfo.StartTime.IsZero() {
-		baseInfo.StartTime = newEntry.Timestamp
-	}
-	newEntry.ServiceInformation = baseInfo
-
-	return newEntry, nil
-}

parser/ts_parser.go

@@ -1,136 +0,0 @@
-package parser
-
-import (
-	"log/slog"
-	"regexp"
-	"strconv"
-	"strings"
-	"tixel_watch/helpers"
-	"tixel_watch/models"
-)
-
-var (
-	tsServicePattern    = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
-	tsTransferIDPattern = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
-	tsDetailPattern1    = regexp.MustCompile(`in: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<dest>\S+) sender-id=(?P<sender_id>\S+)`)
-	tsDetailPattern2    = regexp.MustCompile(`out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s`)
-	tsDetailPattern3    = regexp.MustCompile(`out: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<src>\S+) receiver=(?P<receiver>\S+)`)
-	tsDetailPattern4    = regexp.MustCompile(`out: Start transfer (?P<thread>\d+/\d+), src=(?P<src>[^ ]*) dest=(?P<dest>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)`)
-)
-
-type TSParser struct{}
-
-func (p *TSParser) Parse(line string) (models.LogMessage, error) {
-	newEntry := models.LogMessage{
-		Service: "tixstream",
-	}
-	syslogFields, logContent := helpers.ExtractSyslogHeader(line)
-	newEntry.Host = syslogFields.Hostname
-	newEntry.Raw = line
-	newEntry.Type = "service_log"
-	matches := tsServicePattern.FindStringSubmatch(logContent)
-	if len(matches) > 0 {
-		timestampStr := strings.Join(strings.Split(matches[2], " "), "T")
-		timestamp, err := helpers.ParseRFC3339WithOptionalZ(timestampStr)
-		if err != nil {
-			slog.Error("unable to parse time", "error", err)
-		}
-		if timestamp.IsZero() {
-			timestamp = syslogFields.SysLogTimestamp
-		}
-		newEntry.LogLevel = strings.TrimSpace(matches[1])
-		newEntry.LogLevel = strings.ReplaceAll(newEntry.LogLevel, "ACE_Message_Block", "")
-		newEntry.Timestamp = timestamp
-		newEntry.LogMessage = strings.TrimSpace(matches[3])
-	} else {
-		newEntry.LogMessage = logContent
-	}
-	var baseInfo models.TSTransferInfo
-	trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
-	var transferID string
-	if len(trNameMatch) > 0 {
-		transferID = trNameMatch[1]
-		newEntry.LogMessage = trNameMatch[2]
-		split := strings.Fields(trNameMatch[2])
-		switch split[0] {
-		case "in:":
-			baseInfo.Direction = "incoming"
-		case "out:":
-			baseInfo.Direction = "outgoing"
-		}
-	}
-
-	msg := strings.ReplaceAll(logContent, "    ", " ")
-	parts := strings.Fields(msg)
-
-	if len(parts) < 5 {
-		return newEntry, nil
-	}
-	tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
-	if len(tsDetail) > 0 {
-		threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
-		baseInfo.Lane = threadInt
-		buffersInt, _ := strconv.Atoi(tsDetail[2])
-		fileCountInt, _ := strconv.Atoi(tsDetail[3])
-		fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
-		streamsInt, _ := strconv.Atoi(tsDetail[6])
-		datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
-		chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
-		baseInfo.Buffers = buffersInt
-		baseInfo.FileCount = fileCountInt
-		baseInfo.FileSizeMB = fileSizeFloat
-		baseInfo.ChunkSize = chunkSizeInt
-		baseInfo.Streams = streamsInt
-		baseInfo.TargetDatarate = datarateFloat
-		baseInfo.Protocoll = tsDetail[8]
-		baseInfo.Dest = tsDetail[9]
-		baseInfo.SenderID = tsDetail[10]
-	}
-	tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
-	if len(tsDetail) > 0 {
-		baseInfo.Target = tsDetail[1]
-	}
-	tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
-	if len(tsDetail) > 0 {
-		threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
-		baseInfo.Lane = threadInt
-		buffersInt, _ := strconv.Atoi(tsDetail[2])
-		baseInfo.Buffers = buffersInt
-		fileCountInt, _ := strconv.Atoi(tsDetail[3])
-		fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
-		chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
-		streamsInt, _ := strconv.Atoi(tsDetail[6])
-		datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
-		baseInfo.FileCount = fileCountInt
-		baseInfo.FileSizeMB = fileSizeFloat
-		baseInfo.ChunkSize = chunkSizeInt
-		baseInfo.Streams = streamsInt
-		baseInfo.TargetDatarate = datarateFloat
-		baseInfo.Protocoll = tsDetail[8]
-		baseInfo.Src = tsDetail[9]
-		baseInfo.Receiver = tsDetail[10]
-	}
-	tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
-	if len(tsDetail) > 0 {
-		threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
-		baseInfo.Lane = threadInt
-		baseInfo.Src = tsDetail[2]
-		baseInfo.Dest = tsDetail[3]
-	}
-	if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
-		baseInfo.StartTime = newEntry.Timestamp
-	}
-	if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
-		baseInfo.EndTime = newEntry.Timestamp
-	}
-	if transferID != "" {
-		baseInfo.TransferID = transferID
-	} else {
-		baseInfo.TransferID = "no_transfer_id"
-	}
-	if !baseInfo.StartTime.IsZero() {
-		newEntry.ServiceInformation = baseInfo
-	}
-
-	return newEntry, nil
-}

patterns/repository.go

@@ -0,0 +1,87 @@
+package patterns
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"sync"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Config struct {
+	Patterns map[string]ServiceConfig `yaml:"patterns"`
+}
+
+type ServiceConfig struct {
+	Extractors []ExtractorConfig `yaml:"extractors"`
+}
+
+type ExtractorConfig struct {
+	Name   string            `yaml:"name"`
+	Regex  string            `yaml:"regex"`
+	Fields map[string]string `yaml:"fields"`
+}
+
+type CompiledExtractor struct {
+	Name    string
+	Pattern *regexp.Regexp
+	Fields  map[string]string
+}
+
+type Repository struct {
+	services map[string][]CompiledExtractor
+	mu       sync.RWMutex
+}
+
+var (
+	instance *Repository
+	once     sync.Once
+)
+
+func GetInstance() *Repository {
+	once.Do(func() {
+		instance = &Repository{
+			services: make(map[string][]CompiledExtractor),
+		}
+	})
+	return instance
+}
+
+func (r *Repository) Load(path string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read patterns file: %w", err)
+	}
+
+	var cfg Config
+	if err := yaml.Unmarshal(data, &cfg); err != nil {
+		return fmt.Errorf("failed to parse yaml: %w", err)
+	}
+
+	for service, svcCfg := range cfg.Patterns {
+		var compiledList []CompiledExtractor
+		for _, ext := range svcCfg.Extractors {
+			re, err := regexp.Compile(ext.Regex)
+			if err != nil {
+				return fmt.Errorf("invalid regex in service %s extractor %s: %w", service, ext.Name, err)
+			}
+			compiledList = append(compiledList, CompiledExtractor{
+				Name:    ext.Name,
+				Pattern: re,
+				Fields:  ext.Fields,
+			})
+		}
+		r.services[service] = compiledList
+	}
+	return nil
+}
+
+func (r *Repository) GetExtractors(service string) []CompiledExtractor {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.services[service]
+}

patterns/repository_test.go

@@ -0,0 +1,51 @@
+package patterns
+
+import (
+	"os"
+	"testing"
+)
+
+func TestRepository_Load(t *testing.T) {
+	content := `
+patterns:
+  test_service:
+    extractors:
+      - name: "test_pattern"
+        regex: '^Test (?P<id>\d+) (?P<value>\d+\.\d+)$'
+        fields:
+          id: "int"
+          value: "float"
+`
+	tmpfile, err := os.CreateTemp("", "patterns_test_*.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(content)); err != nil {
+		t.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	repo := GetInstance()
+	err = repo.Load(tmpfile.Name())
+	if err != nil {
+		t.Fatalf("Failed to load repository: %v", err)
+	}
+
+	extractors := repo.GetExtractors("test_service")
+	if len(extractors) != 1 {
+		t.Errorf("Expected 1 extractor, got %d", len(extractors))
+	}
+
+	ext := extractors[0]
+	if ext.Name != "test_pattern" {
+		t.Errorf("Expected name 'test_pattern', got '%s'", ext.Name)
+	}
+
+	if !ext.Pattern.MatchString("Test 123 45.67") {
+		t.Error("Regex did not match valid string")
+	}
+}

service_monitor.go

@@ -2,26 +2,32 @@ package main
 
 import (
 	"bufio"
+	"codeberg.org/pata1704/drain3"
 	"context"
 	"encoding/json"
 	"fmt"
 	"log/slog"
 	"os/exec"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
-	"tixel_watch/parser"
+	"watch-tool/parser"
 )
 
 type ServiceMonitor struct {
-	config ServiceConfig
+	config      ServiceConfig
+	hostname    string
+	drainConfig *drain3.Config
+	stateDir    string
 }
 
-func NewServiceMonitor(config ServiceConfig) *ServiceMonitor {
+func NewServiceMonitor(config ServiceConfig, hostname string, drainCfg *drain3.Config, stateDir string) *ServiceMonitor {
 	return &ServiceMonitor{
-		config: config,
+		config:      config,
+		hostname:    hostname,
+		drainConfig: drainCfg,
+		stateDir:    stateDir,
 	}
 }
 
@@ -50,7 +56,8 @@ func (sm *ServiceMonitor) Start(ctx context.Context, out chan<- models.LogMessag
 		}
 	}()
 
-	parser := NewJournalEntryParser(sm.config.Name, sm.config.Service)
+	jParser := NewJournalEntryParser(sm.config.Name, sm.config.Service, sm.hostname, sm.drainConfig, sm.stateDir)
+	defer jParser.Close()
 
 	for scanner.Scan() {
 		select {
@@ -64,7 +71,7 @@ func (sm *ServiceMonitor) Start(ctx context.Context, out chan<- models.LogMessag
 			continue
 		}
 
-		entry, err := parser.Parse(line)
+		entry, err := jParser.Parse(line)
 		if err != nil {
 			slog.Error("error parsing journal entry", "service", sm.config.Name, "error", err)
 			continue
@@ -109,22 +116,46 @@ func (sm *ServiceMonitor) buildJournalctlArgs() []string {
 type JournalEntryParser struct {
 	serviceName string
 	unitName    string
+	hostname    string
+	innerParser parser.Parser
 }
 
-func NewJournalEntryParser(serviceName, unitName string) *JournalEntryParser {
+func NewJournalEntryParser(serviceName, unitName, hostname string, drainCfg *drain3.Config, stateDir string) *JournalEntryParser {
+	pCfg := parser.ParserConfig{
+		ServiceName: serviceName,
+		LogType:     "custom",
+		Hostname:    hostname,
+		DrainConfig: drainCfg,
+		StateDir:    stateDir,
+	}
+
+	inner, err := parser.New(pCfg)
+	if err != nil {
+		slog.Error("Failed to create inner parser for service", "service", serviceName, "error", err)
+	}
+
 	return &JournalEntryParser{
 		serviceName: serviceName,
 		unitName:    unitName,
+		hostname:    hostname,
+		innerParser: inner,
 	}
 }
 
+func (jep *JournalEntryParser) Close() error {
+	if jep.innerParser != nil {
+		return jep.innerParser.Close()
+	}
+	return nil
+}
+
 func (jep *JournalEntryParser) Parse(jsonLine string) (models.LogMessage, error) {
 	var journalData map[string]any
 	if err := json.Unmarshal([]byte(jsonLine), &journalData); err != nil {
 		return models.LogMessage{}, fmt.Errorf("JSON unmarshal error: %w", err)
 	}
 
-	entry := models.NewLogMessage("service_log", hostname)
+	entry := models.NewLogMessage("service_log", jep.hostname)
 	entry.Service = jep.serviceName
 	entry.Unit = jep.unitName
 
@@ -166,11 +197,25 @@ func (jep *JournalEntryParser) Parse(jsonLine string) (models.LogMessage, error)
 
 	entry.Raw = jsonLine
 
-	entry = jep.parseServiceSpecific(entry)
+	if jep.innerParser != nil && entry.LogMessage != "" {
+		parsedMsg, err := jep.innerParser.Parse(entry.LogMessage)
+		if err == nil {
+			jep.mergeEntries(&entry, &parsedMsg)
+		}
+	}
 
 	return entry, nil
 }
 
+func (jep *JournalEntryParser) mergeEntries(target *models.LogMessage, source *models.LogMessage) {
+	for k, v := range source.Fields {
+		target.Fields[k] = v
+	}
+	if source.LogLevel != "" {
+		target.LogLevel = source.LogLevel
+	}
+}
+
 func (jep *JournalEntryParser) getPriorityName(priority string) string {
 	priorityNames := map[string]string{
 		"0": "emergency",
@@ -202,314 +247,10 @@ func (jep *JournalEntryParser) extractSystemdFields(journalData map[string]any,
 	for _, field := range systemdFields {
 		if value, ok := journalData[field]; ok {
 			esFieldName := strings.ToLower(strings.TrimPrefix(field, "_"))
+			if entry.Fields == nil {
+				entry.Fields = make(map[string]any)
+			}
 			entry.Fields[esFieldName] = value
 		}
 	}
 }
-
-func (jep *JournalEntryParser) parseServiceSpecific(entry models.LogMessage) models.LogMessage {
-	logParser, err := parser.New(jep.serviceName, "custom")
-	if err != nil {
-		slog.Error("cannot get service specific parser")
-		return entry
-	}
-	entry, err = logParser.Parse(entry.LogMessage)
-	return entry
-}
-
-var (
-	amServicePattern       = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
-	tccServicePattern      = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(\w+)\s+(\d+)\s+---\s+\[\s*([^\]]*)\]\s+([\w\.]+)\s*:\s*(.*)$`)
-	tjmServicePattern      = regexp.MustCompile(`^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?<level>\S+)\s+(?<pid>\d+).*?\[(?<collatation_id>[^\]]*)\]\s+\[(?<username>[^\]]*)\]\s+\[(?<thread>[^\]]*)\]\s+(?<class>.*?)\s+:\s+(?<message>.*)`)
-	tjmTransferNamePattern = regexp.MustCompile(`^(\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (.*)$`)
-	tsServicePattern       = regexp.MustCompile(`^(?<level>\S+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?<message>.*)`)
-	tsTransferIDPattern    = regexp.MustCompile(`^(?<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?<message>.*)`)
-	tjmTransferIDPattern1  = regexp.MustCompile(`(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
-	tjmTransferIDPattern2  = regexp.MustCompile(`(?P<before>.*)(?P<transfer>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)`)
-	tsDetailPattern1       = regexp.MustCompile(`in: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<dest>\S+) sender-id=(?P<sender_id>\S+)`)
-	tsDetailPattern2       = regexp.MustCompile(`out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s`)
-	tsDetailPattern3       = regexp.MustCompile(`out: Transfer start (?P<thread>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<files>\d+) size=(?P<size>[0-9.]+) MByte chunksize=(?P<chunksize>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_datarate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<src>\S+) receiver=(?P<receiver>\S+)`)
-	tsDetailPattern4       = regexp.MustCompile(`out: Start transfer (?P<thread>\d+/\d+), src=(?P<src>[^ ]*) dest=(?P<dest>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)`)
-	nginxAccessPattern     = regexp.MustCompile(`^(\S+)\s+\S+\s+(\S+)\s+\[([^\]]+)\]\s+"([^"]+)"\s+(\d+)\s+(\d+|-)\s*(?:"([^"]*)"\s+"([^"]*)")?`)
-)
-
-// func parseTixstreamService(entry models.LogMessage) models.LogMessage {
-// 	newEntry := entry
-// 	var baseInfo models.TSTransferInfo
-
-// 	matches := tsServicePattern.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(matches) > 0 {
-// 		timestamp := strings.Join(strings.Split(matches[2], " "), "T")
-// 		newEntry.LogLevel = strings.TrimSpace(matches[1])
-// 		if newEntry.Timestamp.IsZero() {
-// 			timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
-// 			if err != nil {
-// 				slog.Error("cant parse time string", "error", err)
-// 			}
-// 			newEntry.Timestamp = timeParsed
-// 		}
-// 		newEntry.LogMessage = strings.TrimSpace(matches[3])
-// 	}
-// 	trNameMatch := tsTransferIDPattern.FindStringSubmatch(newEntry.LogMessage)
-// 	var transferID string
-// 	if len(trNameMatch) > 0 {
-// 		transferID = trNameMatch[1]
-// 		newEntry.LogMessage = trNameMatch[2]
-// 		split := strings.Fields(trNameMatch[2])
-// 		switch split[0] {
-// 		case "in:":
-// 			baseInfo.Direction = "incoming"
-// 		case "out:":
-// 			baseInfo.Direction = "outgoing"
-// 		}
-// 	}
-
-// 	msg := strings.ReplaceAll(newEntry.LogMessage, "    ", " ")
-// 	parts := strings.Fields(msg)
-
-// 	if len(parts) < 5 {
-// 		return newEntry
-// 	}
-// 	tsDetail := tsDetailPattern1.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(tsDetail) > 0 {
-// 		threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
-// 		buffersInt, _ := strconv.Atoi(tsDetail[2])
-// 		fileCountInt, _ := strconv.Atoi(tsDetail[3])
-// 		chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
-// 		streamsInt, _ := strconv.Atoi(tsDetail[6])
-// 		datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
-// 		fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
-// 		baseInfo.Lane = threadInt
-// 		baseInfo.Buffers = buffersInt
-// 		baseInfo.FileCount = fileCountInt
-// 		baseInfo.FileSizeMB = fileSizeFloat
-// 		baseInfo.ChunkSize = chunkSizeInt
-// 		baseInfo.Streams = streamsInt
-// 		baseInfo.TargetDatarate = datarateFloat
-// 		baseInfo.Protocoll = tsDetail[8]
-// 		baseInfo.Dest = tsDetail[9]
-// 		baseInfo.SenderID = tsDetail[10]
-// 	}
-// 	tsDetail = tsDetailPattern2.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(tsDetail) > 0 {
-// 		baseInfo.Target = tsDetail[1]
-// 	}
-// 	tsDetail = tsDetailPattern3.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(tsDetail) > 0 {
-// 		threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
-// 		buffersInt, _ := strconv.Atoi(tsDetail[2])
-// 		fileCountInt, _ := strconv.Atoi(tsDetail[3])
-// 		fileSizeFloat, _ := strconv.ParseFloat(tsDetail[4], 64)
-// 		chunkSizeInt, _ := strconv.Atoi(tsDetail[5])
-// 		streamsInt, _ := strconv.Atoi(tsDetail[6])
-// 		datarateFloat, _ := strconv.ParseFloat(tsDetail[7], 64)
-// 		baseInfo.Lane = threadInt
-// 		baseInfo.Buffers = buffersInt
-// 		baseInfo.FileCount = fileCountInt
-// 		baseInfo.FileSizeMB = fileSizeFloat
-// 		baseInfo.ChunkSize = chunkSizeInt
-// 		baseInfo.Streams = streamsInt
-// 		baseInfo.TargetDatarate = datarateFloat
-// 		baseInfo.Protocoll = tsDetail[8]
-// 		baseInfo.Src = tsDetail[9]
-// 		baseInfo.Receiver = tsDetail[10]
-// 	}
-// 	tsDetail = tsDetailPattern4.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(tsDetail) > 0 {
-// 		threadInt, _ := strconv.Atoi(strings.Split(tsDetail[1], "/")[0])
-// 		baseInfo.Lane = threadInt
-// 		baseInfo.Src = tsDetail[2]
-// 		baseInfo.Dest = tsDetail[3]
-// 	}
-// 	if strings.Contains(newEntry.LogMessage, "Transfer start") || strings.Contains(newEntry.LogMessage, "Transfer started,") {
-// 		baseInfo.StartTime = newEntry.Timestamp
-// 	} else {
-// 		baseInfo.StartTime = time.Now()
-// 	}
-// 	if strings.Contains(newEntry.LogMessage, "Transfer stopped local state=finished") {
-// 		baseInfo.EndTime = newEntry.Timestamp
-// 	} else {
-// 		baseInfo.EndTime = baseInfo.StartTime
-// 	}
-// 	if transferID != "" {
-// 		baseInfo.TransferID = transferID
-// 	} else {
-// 		baseInfo.TransferID = "no_transfer_id"
-// 	}
-// 	newEntry.ServiceInformation = baseInfo
-// 	return newEntry
-// }
-
-// func parseTJMService(entry models.LogMessage) models.LogMessage {
-// 	newEntry := entry
-// 	var baseInfo models.TJMTransferInfo
-
-// 	logContent := entry.LogMessage
-// 	msg := strings.TrimSpace(logContent)
-// 	msg = strings.ReplaceAll(msg, "           ", " ")
-// 	msg = strings.ReplaceAll(msg, "---", "")
-// 	msg = strings.ReplaceAll(msg, "  ", " ")
-// 	parts := strings.Fields(msg)
-// 	if len(parts) < 4 {
-// 		return newEntry
-// 	}
-// 	matches := tjmServicePattern.FindStringSubmatch(logContent)
-// 	if len(matches) > 0 {
-// 		timestamp := strings.Join(strings.Split(matches[2], " "), "T")
-// 		newEntry.LogLevel = strings.TrimSpace(matches[1])
-// 		if newEntry.Timestamp.IsZero() {
-// 			timeParsed, err := parseRFC3339WithOptionalZ(timestamp)
-// 			if err != nil {
-// 				slog.Error("cant parse time string", "error", err)
-// 			}
-// 			newEntry.Timestamp = timeParsed
-// 		}
-// 		newEntry.LogLevel = strings.TrimSpace(matches[2])
-// 		newEntry.LogMessage = strings.TrimSpace(matches[8])
-// 		baseInfo = models.TJMTransferInfo{
-// 			ProcessID:     strings.TrimSpace(matches[3]),
-// 			CorrelationID: strings.TrimSpace(matches[4]),
-// 			Username:      strings.TrimSpace(matches[5]),
-// 			ThreadID:      strings.TrimSpace(matches[6]),
-// 			JavaClass:     strings.TrimSpace(matches[7]),
-// 		}
-// 	} else {
-// 		newEntry.LogMessage = logContent
-// 	}
-// 	trNameMatch := tjmTransferNamePattern.FindStringSubmatch(newEntry.LogMessage)
-// 	var transferName string
-// 	var transferID string
-// 	if len(trNameMatch) > 0 {
-// 		transferName = trNameMatch[1]
-// 		newEntry.LogMessage = trNameMatch[2]
-// 		if strings.Contains(trNameMatch[1], "-in") {
-// 			baseInfo.Direction = "incoming"
-// 		}
-// 		if strings.Contains(trNameMatch[1], "-out") {
-// 			baseInfo.Direction = "outgoing"
-// 		}
-// 	}
-// 	trIDMatch := tjmTransferIDPattern1.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(trIDMatch) > 0 {
-// 		transferID = trIDMatch[1]
-// 	}
-// 	trIDMatch = tjmTransferIDPattern2.FindStringSubmatch(newEntry.LogMessage)
-// 	if len(trIDMatch) > 0 {
-// 		transferID = trIDMatch[2]
-// 	}
-// 	if transferID != "" {
-// 		baseInfo.TransferID = transferID
-// 	} else if transferName != "" {
-// 		baseInfo.TransferID = transferName
-// 	} else {
-// 		baseInfo.TransferID = "no_transfer_id"
-// 	}
-// 	baseInfo.StartTime = newEntry.Timestamp
-// 	baseInfo.StartTime = newEntry.Timestamp
-// 	newEntry.ServiceInformation = baseInfo
-
-// 	return newEntry
-// }
-
-// func parseAMService(entry models.LogMessage) models.LogMessage {
-// 	newEntry := entry
-// 	logContent := newEntry.LogMessage
-
-// 	matches := amServicePattern.FindStringSubmatch(strings.TrimSpace(logContent))
-// 	if len(matches) != 7 {
-// 		return newEntry
-// 	}
-// 	timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
-// 	if newEntry.Timestamp.IsZero() {
-// 		timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
-// 		if err != nil {
-// 			slog.Error("cant parse time string", "error", err)
-// 		}
-// 		newEntry.Timestamp = timeParsed
-// 	}
-// 	baseInfo := models.AMBaseInfo{
-// 		ProcessID:  matches[3],
-// 		ThreadID:   strings.TrimSpace(matches[4]),
-// 		LoggerName: matches[5],
-// 	}
-// 	newEntry.LogLevel = matches[2]
-// 	newEntry.LogMessage = matches[6]
-// 	newEntry.ServiceInformation = baseInfo
-
-// 	return newEntry
-// }
-
-// func parseTCCService(entry models.LogMessage) models.LogMessage {
-// 	newEntry := entry
-// 	logContent := newEntry.LogMessage
-
-// 	matches := tccServicePattern.FindStringSubmatch(logContent)
-// 	if len(matches) != 7 {
-// 		return newEntry
-// 	}
-// 	timestampStr := strings.Join(strings.Split(matches[1], " "), "T")
-// 	if newEntry.Timestamp.IsZero() {
-// 		timeParsed, err := parseRFC3339WithOptionalZ(timestampStr)
-// 		if err != nil {
-// 			slog.Error("cant parse time string", "error", err)
-// 		}
-// 		newEntry.Timestamp = timeParsed
-// 	}
-// 	baseInfo := models.TCCBaseInfo{
-// 		ProcessID:  matches[3],
-// 		ThreadID:   strings.TrimSpace(matches[4]),
-// 		LoggerName: matches[5],
-// 	}
-// 	newEntry.LogLevel = matches[2]
-// 	newEntry.LogMessage = matches[6]
-// 	newEntry.ServiceInformation = baseInfo
-
-// 	return newEntry
-// }
-
-// func parseNginxService(entry models.LogMessage) models.LogMessage {
-// 	newEntry := entry
-
-// 	matches := nginxAccessPattern.FindStringSubmatch(strings.TrimSpace(entry.LogMessage))
-// 	if len(matches) < 7 {
-// 		return newEntry
-// 	}
-// 	statusCode, err := strconv.ParseInt(matches[5], 10, 64)
-// 	if err != nil {
-// 		slog.Error("cant parse statuscode", "error", err)
-// 	}
-// 	bytesSend, err := strconv.ParseInt(matches[6], 10, 64)
-// 	if err != nil {
-// 		slog.Error("cant parse bytessend", "error", err)
-// 	}
-// 	baseInfo := models.NGinXBaseInfo{
-// 		ClientIP:   matches[1],
-// 		RemoteUser: matches[2],
-// 		Request:    matches[4],
-// 		StatusCode: int(statusCode),
-// 		BytesSend:  int(bytesSend),
-// 	}
-
-// 	if len(matches) > 7 && matches[7] != "" {
-// 		baseInfo.Referer = matches[7]
-// 	}
-// 	if len(matches) > 8 && matches[8] != "" {
-// 		baseInfo.UserAgent = matches[8]
-// 	}
-
-// 	if requestParts := strings.Fields(matches[4]); len(requestParts) >= 3 {
-// 		baseInfo.HTTPMethod = requestParts[0]
-// 		baseInfo.RequestURI = requestParts[1]
-// 		baseInfo.HTTPVersion = requestParts[2]
-// 	}
-// 	newEntry.ServiceInformation = baseInfo
-
-// 	return newEntry
-// }
-
-// func parseRFC3339WithOptionalZ(timeStr string) (time.Time, error) {
-// 	if !strings.HasSuffix(timeStr, "Z") && !strings.ContainsAny(timeStr[len(timeStr)-6:], "+-") {
-// 		timeStr += "Z"
-// 	}
-// 	return time.Parse(time.RFC3339Nano, timeStr)
-// }

storage_interface.go

@@ -3,7 +3,7 @@ package main
 import (
 	"context"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
 )
 
 type StorageInterface interface {
@@ -25,8 +25,3 @@ type StorageQuery struct {
 	OrderBy   string
 	OrderDesc bool
 }
-
-type ExporterInterface interface {
-	Export(ctx context.Context, entries []models.LogMessage) error
-	HealthCheck(ctx context.Context) error
-}

system_metrics.go

@@ -12,9 +12,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
-	"tixel_watch/models"
+	"watch-tool/models"
 
-	"github.com/elastic/go-elasticsearch/v8"
 	"github.com/shirou/gopsutil/cpu"
 	"github.com/shirou/gopsutil/disk"
 	"github.com/shirou/gopsutil/host"
@@ -31,24 +30,24 @@ type SystemMetricsCollector struct {
 	lastNetworkStats map[string]models.NetworkStat
 	lastDiskStats    map[string]models.DiskIOStat
 	lastMeasureTime  time.Time
+	hostname         string
 }
 
-func NewSystemMetricsCollector(config SystemMetrics, pollInterval int) *SystemMetricsCollector {
+func NewSystemMetricsCollector(config SystemMetrics, pollInterval int, hostname string) *SystemMetricsCollector {
 	return &SystemMetricsCollector{
 		config:           config,
 		pollInterval:     pollInterval,
 		lastNetworkStats: make(map[string]models.NetworkStat),
 		lastDiskStats:    make(map[string]models.DiskIOStat),
 		lastMeasureTime:  time.Now(),
+		hostname:         hostname,
 	}
 }
 
-func (smc *SystemMetricsCollector) Start(ctx context.Context, es *elasticsearch.Client, baseIndex string) {
+func (smc *SystemMetricsCollector) Start(ctx context.Context, storage StorageInterface, logChan chan<- models.LogMessage) {
 	ticker := time.NewTicker(time.Duration(smc.pollInterval) * time.Second)
 	defer ticker.Stop()
 
-	sender := NewElasticsearchSender(es)
-
 	for {
 		select {
 		case <-ctx.Done():
@@ -61,15 +60,23 @@ func (smc *SystemMetricsCollector) Start(ctx context.Context, es *elasticsearch.
 				continue
 			}
 
-			if err := sender.SendSystemMetrics(baseIndex, metrics); err != nil {
+			entry := models.NewLogMessage("system_metrics", smc.hostname)
-				slog.Error("error sending system metrics", "error", err)
+			entry.Service = "system-metrics"
+			entry.LogLevel = "Info"
+			entry.SystemMetrics = metrics
+
+			select {
+			case logChan <- entry:
+			case <-ctx.Done():
+				return
+			default:
+				slog.Warn("Log channel is full, system metrics dropped")
 			}
 		}
 	}
 }
-
 func (smc *SystemMetricsCollector) collectMetrics() (models.SystemResources, error) {
-	result := models.NewSystemResources(hostname)
+	result := models.NewSystemResources(smc.hostname)
 
 	var err error
 

system_metricsV2.go

@@ -1,40 +0,0 @@
-package main
-
-import (
-	"context"
-	"log/slog"
-	"time"
-	"tixel_watch/models"
-)
-
-func (smc *SystemMetricsCollector) StartV2(ctx context.Context, storage StorageInterface, logChan chan<- models.LogMessage) {
-	ticker := time.NewTicker(time.Duration(smc.pollInterval) * time.Second)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			slog.Info("System metrics collector stopped")
-			return
-		case <-ticker.C:
-			metrics, err := smc.collectMetrics()
-			if err != nil {
-				slog.Error("error collecting system metrics", "error", err)
-				continue
-			}
-
-			entry := models.NewLogMessage("system_metrics", hostname)
-			entry.Service = "system-metrics"
-			entry.LogLevel = "Info"
-			entry.SystemMetrics = metrics
-
-			select {
-			case logChan <- entry:
-			case <-ctx.Done():
-				return
-			default:
-				slog.Warn("Log channel is full, system metrics dropped")
-			}
-		}
-	}
-}

tixel-watch.service

@@ -1,19 +0,0 @@
-[Unit]
-Description=tixel-watch
-After=network.target
-
-[Service]
-Type=simple
-User=tixstream
-Group=tixstream
-WorkingDirectory=/opt/tixel/tixel-watch
-ExecStart=/opt/tixel/tixel-watch/tixel-watch
-Restart=on-failure
-RestartSec=5
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=tixel-watch
-
-[Install]
-WantedBy=multi-user.target
-

web_service.go

@@ -7,35 +7,46 @@ import (
 	"log/slog"
 	"net/http"
 	"os/exec"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
-
+	"watch-tool/models"
-	"github.com/elastic/go-elasticsearch/v8"
 )
 
 type WebService struct {
-	server   *http.Server
+	server  *http.Server
-	esClient *elasticsearch.Client
+	storage StorageInterface
-	config   *Config
+	config  *Config
 }
 
-func NewWebService(config *Config, esClient *elasticsearch.Client) *WebService {
+func LoggingMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		slog.Debug("WebService", "Remote-Address", r.RemoteAddr, "Method", r.Method, "Path", r.URL.Path)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func NewWebService(config *Config, storage StorageInterface) *WebService {
 	mux := http.NewServeMux()
 
 	ws := &WebService{
-		esClient: esClient,
+		storage: storage,
-		config:   config,
+		config:  config,
 	}
 
-	mux.HandleFunc("/export", ws.handleExport)
+	mux.HandleFunc("GET /health", ws.handleHealth)
-	mux.HandleFunc("/health", ws.handleHealth)
+	mux.HandleFunc("GET /logs", ws.handleLogs)
-	mux.HandleFunc("/indices", ws.handleIndices)
+	mux.HandleFunc("GET /export", ws.handleExport)
+	mux.HandleFunc("GET /stats", ws.handleStats)
+	mux.HandleFunc("GET /stats/{service}", ws.handleServiceStats)
+
+	loggedMux := LoggingMiddleware(mux)
 
 	addr := fmt.Sprintf("%s:%d", config.WebService.Host, config.WebService.Port)
 	ws.server = &http.Server{
 		Addr:         addr,
-		Handler:      mux,
+		Handler:      loggedMux,
 		ReadTimeout:  30 * time.Second,
 		WriteTimeout: 300 * time.Second,
 		IdleTimeout:  60 * time.Second,
@@ -64,67 +75,30 @@ func (ws *WebService) Start(ctx context.Context) error {
 	return nil
 }
 
-func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
-	indices := ws.parseIndicesParam(r)
-	if len(indices) == 0 {
-		http.Error(w, "No indices specified. Use ?indices=index1,index2", http.StatusBadRequest)
-		return
-	}
-
-	size := ws.parseSizeParam(r)
-	since := ws.parseSinceParam(r)
-
-	slog.Info("Export request received", "indices", indices, "size", size, "since", since)
-
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Content-Disposition", "attachment; filename=elasticsearch_export.json")
-
-	exporter := NewElasticsearchExporter(ws.esClient)
-	if err := exporter.ExportToStream(r.Context(), indices, size, since, w); err != nil {
-		slog.Error("export error", "error", err)
-		http.Error(w, fmt.Sprintf("Export error: %v", err), http.StatusInternalServerError)
-		return
-	}
-
-	slog.Info("Export completed successfully", "indices", indices)
-}
-
 func (ws *WebService) handleHealth(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
+
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	status := map[string]any{
-		return
+		"status":    "healthy",
+		"timestamp": time.Now(),
+		"storage":   "sqlite",
 	}
 
 	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
 	defer cancel()
 
-	res, err := ws.esClient.Info(ws.esClient.Info.WithContext(ctx))
+	_, err := ws.storage.Query(ctx, StorageQuery{
+		Limit: 1,
+	})
+
 	if err != nil {
+		status["storage_status"] = "unhealthy"
+		status["storage_error"] = err.Error()
 		w.WriteHeader(http.StatusServiceUnavailable)
-		json.NewEncoder(w).Encode(map[string]any{
+	} else {
-			"status": "unhealthy",
+		status["storage_status"] = "healthy"
-			"error":  err.Error(),
-		})
-		return
 	}
-	res.Body.Close()
-
-	if res.IsError() {
-		w.WriteHeader(http.StatusServiceUnavailable)
-		json.NewEncoder(w).Encode(map[string]any{
-			"status": "unhealthy",
-			"error":  res.String(),
-		})
-		return
-	}
-
 	statusMap := make(map[string]any)
-	statusMap["elasticsearch"] = map[string]any{"status": "healthy", "timestamp": time.Now()}
+	statusMap["watch"] = status
 
 	for _, service := range ws.config.Services {
 		statusCommand := []string{"sudo", "systemctl", "status", service.Name, "--no-pager"}
@@ -150,89 +124,295 @@ func (ws *WebService) handleHealth(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(statusMap)
 }
 
-func (ws *WebService) handleIndices(w http.ResponseWriter, r *http.Request) {
+func (ws *WebService) handleLogs(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
+
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	query := ws.parseLogsQuery(r)
+
+	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	defer cancel()
+
+	entries, err := ws.storage.Query(ctx, query)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Query error: %v", err), http.StatusInternalServerError)
 		return
 	}
 
+	response := map[string]any{
+		"entries": entries,
+		"count":   len(entries),
+		"query":   query,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(response)
+}
+
+func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
+
+	query := ws.parseLogsQuery(r)
+
+	ctx, cancel := context.WithTimeout(r.Context(), 300*time.Second)
+	defer cancel()
+
+	entries, err := ws.storage.Query(ctx, query)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Export query error: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	filename := fmt.Sprintf("watch_export_%s.json", time.Now().Format("20060102_150405"))
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
+
+	exportData := map[string]any{
+		"export_info": map[string]any{
+			"timestamp":   time.Now(),
+			"entry_count": len(entries),
+			"query":       query,
+			"exported_by": "watch",
+		},
+		"entries": entries,
+	}
+
+	if err := json.NewEncoder(w).Encode(exportData); err != nil {
+		slog.Error("Failed to encode export data", "error", err)
+		return
+	}
+
+	slog.Info("Data exported", "count", len(entries), "query", query)
+}
+
+func (ws *WebService) handleStats(w http.ResponseWriter, r *http.Request) {
+
 	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
 	defer cancel()
 
-	res, err := ws.esClient.Cat.Indices(
+	allEntries, err := ws.storage.Query(ctx, StorageQuery{})
-		ws.esClient.Cat.Indices.WithContext(ctx),
-		ws.esClient.Cat.Indices.WithFormat("json"),
-	)
 	if err != nil {
-		http.Error(w, fmt.Sprintf("Error fetching indices: %v", err), http.StatusInternalServerError)
+		http.Error(w, fmt.Sprintf("Stats query error: %v", err), http.StatusInternalServerError)
-		return
-	}
-	defer res.Body.Close()
-
-	if res.IsError() {
-		http.Error(w, fmt.Sprintf("Elasticsearch error: %s", res.String()), http.StatusInternalServerError)
 		return
 	}
 
-	var indices []map[string]any
+	recentEntries, err := ws.storage.Query(ctx, StorageQuery{
-	if err := json.NewDecoder(res.Body).Decode(&indices); err != nil {
+		StartTime: time.Now().Add(-time.Hour),
-		http.Error(w, fmt.Sprintf("Error decoding response: %v", err), http.StatusInternalServerError)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(map[string]any{
-		"indices": indices,
-		"count":   len(indices),
 	})
-}
+	if err != nil {
-
+		slog.Error("Failed to query recent entries", "error", err)
-func (ws *WebService) parseIndicesParam(r *http.Request) []string {
+		recentEntries = []models.LogMessage{}
-	indicesParam := r.URL.Query().Get("indices")
-	if indicesParam == "" {
-		return nil
 	}
 
-	indices := strings.Split(indicesParam, ",")
+	stats := map[string]any{
-	var result []string
+		"total_entries":  len(allEntries),
-	for _, index := range indices {
+		"recent_entries": len(recentEntries),
-		index = strings.TrimSpace(index)
+		"timestamp":      time.Now(),
-		if index != "" {
+	}
-			result = append(result, index)
+
+	typeCounts := make(map[string]int)
+	serviceCounts := make(map[string]int)
+	toolCounts := make(map[string]int)
+
+	for _, entry := range allEntries {
+		typeCounts[entry.Type]++
+		if entry.Service != "" {
+			serviceCounts[entry.Service]++
+		}
+		if entry.Tool != "" {
+			toolCounts[entry.Tool]++
 		}
 	}
 
-	return result
+	stats["by_type"] = typeCounts
+	stats["by_service"] = serviceCounts
+	stats["by_tool"] = toolCounts
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(stats)
 }
 
-func (ws *WebService) parseSinceParam(r *http.Request) int {
+func (ws *WebService) handleServiceStats(w http.ResponseWriter, r *http.Request) {
-	sinceParam := r.URL.Query().Get("since")
+	service := r.PathValue("service")
-	if sinceParam == "" {
+	if service == "" {
-		return 0
+		http.Error(w, "Service parameter is missing", http.StatusBadRequest)
+		return
 	}
 
-	since, err := strconv.Atoi(sinceParam)
+	timeRangeStr := r.URL.Query().Get("time_range")
+	var startTime time.Time
+	if timeRangeStr == "" {
+		startTime = time.Now().Add(-24 * time.Hour)
+	} else {
+		duration, err := time.ParseDuration(timeRangeStr)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Invalid time_range: %v", err), http.StatusBadRequest)
+			return
+		}
+		startTime = time.Now().Add(-duration)
+	}
+
+	query := StorageQuery{
+		Service:   service,
+		StartTime: startTime,
+		Limit:     0,
+		OrderDesc: false,
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	defer cancel()
+
+	entries, err := ws.storage.Query(ctx, query)
 	if err != nil {
-		return 0
+		slog.Error("Failed to query service stats", "service", service, "error", err)
+		http.Error(w, fmt.Sprintf("Query error: %v", err), http.StatusInternalServerError)
+		return
 	}
 
-	return since
+	uniqueTransfersTotal := make(map[string]struct{})
+	uniqueTransfersIncoming := make(map[string]struct{})
+	uniqueTransfersOutgoing := make(map[string]struct{})
+	uniqueTransfersNil := make(map[string]struct{})
+
+	for _, entry := range entries {
+		var identifier string
+		var direction string
+
+		if entry.Fields != nil {
+			if id, ok := entry.Fields["transfer_id"].(string); ok {
+				identifier = id
+			} else if id, ok := entry.Fields["correlation_id"].(string); ok {
+				identifier = id
+			}
+
+			if dir, ok := entry.Fields["direction"].(string); ok {
+				direction = dir
+			} else if rawName, ok := entry.Fields["transfer_name_raw"].(string); ok {
+				if strings.Contains(rawName, "-in") {
+					direction = "incoming"
+				} else if strings.Contains(rawName, "-out") {
+					direction = "outgoing"
+				}
+			}
+
+			if direction == "" && entry.Service == "tixstream" {
+				if strings.HasPrefix(entry.Raw, "in:") {
+					direction = "incoming"
+				} else if strings.HasPrefix(entry.Raw, "out:") {
+					direction = "outgoing"
+				}
+			}
+		}
+
+		if identifier == "" && entry.ServiceInformation != nil {
+			switch v := entry.ServiceInformation.(type) {
+			case models.TSTransferInfo:
+				identifier = v.TransferID
+				direction = v.Direction
+			case *models.TSTransferInfo:
+				identifier = v.TransferID
+				direction = v.Direction
+			case models.TJMTransferInfo:
+				identifier = v.TransferID
+				direction = v.Direction
+			case *models.TJMTransferInfo:
+				identifier = v.TransferID
+				direction = v.Direction
+			case map[string]any:
+				identifier, _ = v["transfer_identifier"].(string)
+				direction, _ = v["direction"].(string)
+			}
+		}
+
+		if identifier != "" && identifier != "no_transfer_id" {
+			uniqueTransfersTotal[identifier] = struct{}{}
+
+			dirLower := strings.ToLower(direction)
+
+			if strings.Contains(dirLower, "outgoing") || strings.Contains(dirLower, "out") {
+				uniqueTransfersOutgoing[identifier] = struct{}{}
+			} else if strings.Contains(dirLower, "incoming") || strings.Contains(dirLower, "in") {
+				uniqueTransfersIncoming[identifier] = struct{}{}
+			} else {
+				uniqueTransfersNil[identifier] = struct{}{}
+			}
+		}
+	}
+
+	stats := map[string]any{
+		"service":    service,
+		"start_time": startTime,
+		"end_time":   time.Now(),
+		"transfer_counts": map[string]any{
+			"total":                    len(uniqueTransfersTotal),
+			"incoming":                 len(uniqueTransfersIncoming),
+			"outgoing":                 len(uniqueTransfersOutgoing),
+			"nil_or_unknown_direction": len(uniqueTransfersNil),
+		},
+		"entry_count": len(entries),
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(stats)
 }
 
-func (ws *WebService) parseSizeParam(r *http.Request) int {
+func (ws *WebService) parseLogsQuery(r *http.Request) StorageQuery {
-	sizeParam := r.URL.Query().Get("size")
+	query := StorageQuery{
-	if sizeParam == "" {
+		Limit:     100,
-		return 1000
+		OrderBy:   "timestamp",
+		OrderDesc: true,
 	}
 
-	size, err := strconv.Atoi(sizeParam)
+	if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
-	if err != nil || size <= 0 {
+		if limit, err := strconv.Atoi(limitStr); err == nil && limit > 0 {
-		return 1000
+			if limit > 10000 {
+				limit = 10000
+			}
+			query.Limit = limit
+		}
 	}
 
-	if size > 10000 {
+	if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" {
-		size = 10000
+		if offset, err := strconv.Atoi(offsetStr); err == nil && offset >= 0 {
+			query.Offset = offset
+		}
 	}
 
-	return size
+	if startTime := r.URL.Query().Get("start_time"); startTime != "" {
+		if t, err := time.Parse(time.RFC3339, startTime); err == nil {
+			query.StartTime = t
+		}
+	}
+
+	if endTime := r.URL.Query().Get("end_time"); endTime != "" {
+		if t, err := time.Parse(time.RFC3339, endTime); err == nil {
+			query.EndTime = t
+		}
+	}
+
+	if service := r.URL.Query().Get("service"); service != "" {
+		query.Service = strings.TrimSpace(service)
+	}
+
+	if tool := r.URL.Query().Get("tool"); tool != "" {
+		query.Tool = strings.TrimSpace(tool)
+	}
+
+	if logLevel := r.URL.Query().Get("log_level"); logLevel != "" {
+		query.LogLevel = strings.TrimSpace(logLevel)
+	}
+
+	if entryType := r.URL.Query().Get("type"); entryType != "" {
+		query.Type = strings.TrimSpace(entryType)
+	}
+
+	if orderBy := r.URL.Query().Get("order_by"); orderBy != "" {
+		validFields := []string{"timestamp", "service", "tool", "type", "log_level"}
+		if slices.Contains(validFields, orderBy) {
+			query.OrderBy = orderBy
+		}
+	}
+
+	if orderDesc := r.URL.Query().Get("order_desc"); orderDesc != "" {
+		query.OrderDesc = orderDesc == "true"
+	}
+
+	return query
 }

web_serviceV2.go

@@ -1,309 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"log/slog"
-	"net/http"
-	"os/exec"
-	"slices"
-	"strconv"
-	"strings"
-	"time"
-	"tixel_watch/models"
-)
-
-type WebServiceV2 struct {
-	server  *http.Server
-	storage StorageInterface
-	config  *Config
-}
-
-func LoggingMiddleware(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		slog.Debug("WebService", "Remote-Address", r.RemoteAddr, "Method", r.Method, "Path", r.URL.Path)
-		next.ServeHTTP(w, r)
-	})
-}
-
-func NewWebServiceV2(config *Config, storage StorageInterface) *WebServiceV2 {
-	mux := http.NewServeMux()
-
-	ws := &WebServiceV2{
-		storage: storage,
-		config:  config,
-	}
-
-	mux.HandleFunc("GET /health", ws.handleHealth)
-	mux.HandleFunc("GET /logs", ws.handleLogs)
-	mux.HandleFunc("GET /export", ws.handleExport)
-	mux.HandleFunc("GET /stats", ws.handleStats)
-
-	loggedMux := LoggingMiddleware(mux)
-
-	addr := fmt.Sprintf("%s:%d", config.WebService.Host, config.WebService.Port)
-	ws.server = &http.Server{
-		Addr:         addr,
-		Handler:      loggedMux,
-		ReadTimeout:  30 * time.Second,
-		WriteTimeout: 300 * time.Second,
-		IdleTimeout:  60 * time.Second,
-	}
-
-	return ws
-}
-
-func (ws *WebServiceV2) Start(ctx context.Context) error {
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-
-		if err := ws.server.Shutdown(shutdownCtx); err != nil {
-			slog.Error("web service shutdown error", "error", err)
-		}
-	}()
-
-	slog.Info("Starting web service", "address", ws.server.Addr)
-
-	if err := ws.server.ListenAndServe(); err != http.ErrServerClosed {
-		return fmt.Errorf("web service error: %w", err)
-	}
-
-	return nil
-}
-
-func (ws *WebServiceV2) handleHealth(w http.ResponseWriter, r *http.Request) {
-	// if r.Method != http.MethodGet {
-	// 	http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	// 	return
-	// }
-
-	status := map[string]any{
-		"status":    "healthy",
-		"timestamp": time.Now(),
-		"storage":   "sqlite",
-	}
-
-	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
-	defer cancel()
-
-	_, err := ws.storage.Query(ctx, StorageQuery{
-		Limit: 1,
-	})
-
-	if err != nil {
-		status["storage_status"] = "unhealthy"
-		status["storage_error"] = err.Error()
-		w.WriteHeader(http.StatusServiceUnavailable)
-	} else {
-		status["storage_status"] = "healthy"
-	}
-	statusMap := make(map[string]any)
-	statusMap["tixel-watch"] = status
-
-	for _, service := range ws.config.Services {
-		statusCommand := []string{"sudo", "systemctl", "status", service.Name, "--no-pager"}
-		if service.Enabled {
-			serviceStatus, err := exec.Command(statusCommand[0], statusCommand[1:]...).Output()
-			if err != nil {
-				slog.Error("error executing status command", "error", err)
-				continue
-			}
-			lines := strings.SplitSeq(string(serviceStatus), "\n")
-			for line := range lines {
-				if strings.Contains(line, "Active:") {
-					serviceHealth, found := strings.CutPrefix(strings.TrimSpace(line), "Active:")
-					if found {
-						statusMap[service.Name] = map[string]any{"status": serviceHealth, "timestamp": time.Now()}
-					}
-				}
-			}
-		}
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(statusMap)
-}
-
-func (ws *WebServiceV2) handleLogs(w http.ResponseWriter, r *http.Request) {
-	// if r.Method != http.MethodGet {
-	// 	http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	// 	return
-	// }
-
-	query := ws.parseLogsQuery(r)
-
-	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
-	defer cancel()
-
-	entries, err := ws.storage.Query(ctx, query)
-	if err != nil {
-		http.Error(w, fmt.Sprintf("Query error: %v", err), http.StatusInternalServerError)
-		return
-	}
-
-	response := map[string]any{
-		"entries": entries,
-		"count":   len(entries),
-		"query":   query,
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(response)
-}
-
-func (ws *WebServiceV2) handleExport(w http.ResponseWriter, r *http.Request) {
-	// if r.Method != http.MethodGet {
-	// 	http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	// 	return
-	// }
-
-	query := ws.parseLogsQuery(r)
-
-	ctx, cancel := context.WithTimeout(r.Context(), 300*time.Second)
-	defer cancel()
-
-	entries, err := ws.storage.Query(ctx, query)
-	if err != nil {
-		http.Error(w, fmt.Sprintf("Export query error: %v", err), http.StatusInternalServerError)
-		return
-	}
-
-	filename := fmt.Sprintf("tixel_export_%s.json", time.Now().Format("20060102_150405"))
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
-
-	exportData := map[string]any{
-		"export_info": map[string]any{
-			"timestamp":   time.Now(),
-			"entry_count": len(entries),
-			"query":       query,
-			"exported_by": "tixel-watch",
-		},
-		"entries": entries,
-	}
-
-	if err := json.NewEncoder(w).Encode(exportData); err != nil {
-		slog.Error("Failed to encode export data", "error", err)
-		return
-	}
-
-	slog.Info("Data exported", "count", len(entries), "query", query)
-}
-
-func (ws *WebServiceV2) handleStats(w http.ResponseWriter, r *http.Request) {
-	// if r.Method != http.MethodGet {
-	// 	http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	// 	return
-	// }
-
-	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
-	defer cancel()
-
-	allEntries, err := ws.storage.Query(ctx, StorageQuery{})
-	if err != nil {
-		http.Error(w, fmt.Sprintf("Stats query error: %v", err), http.StatusInternalServerError)
-		return
-	}
-
-	recentEntries, err := ws.storage.Query(ctx, StorageQuery{
-		StartTime: time.Now().Add(-time.Hour),
-	})
-	if err != nil {
-		slog.Error("Failed to query recent entries", "error", err)
-		recentEntries = []models.LogMessage{}
-	}
-
-	stats := map[string]any{
-		"total_entries":  len(allEntries),
-		"recent_entries": len(recentEntries),
-		"timestamp":      time.Now(),
-	}
-
-	typeCounts := make(map[string]int)
-	serviceCounts := make(map[string]int)
-	toolCounts := make(map[string]int)
-
-	for _, entry := range allEntries {
-		typeCounts[entry.Type]++
-		if entry.Service != "" {
-			serviceCounts[entry.Service]++
-		}
-		if entry.Tool != "" {
-			toolCounts[entry.Tool]++
-		}
-	}
-
-	stats["by_type"] = typeCounts
-	stats["by_service"] = serviceCounts
-	stats["by_tool"] = toolCounts
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(stats)
-}
-
-func (ws *WebServiceV2) parseLogsQuery(r *http.Request) StorageQuery {
-	query := StorageQuery{
-		Limit:     100,
-		OrderBy:   "timestamp",
-		OrderDesc: true,
-	}
-
-	if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
-		if limit, err := strconv.Atoi(limitStr); err == nil && limit > 0 {
-			if limit > 10000 {
-				limit = 10000
-			}
-			query.Limit = limit
-		}
-	}
-
-	if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" {
-		if offset, err := strconv.Atoi(offsetStr); err == nil && offset >= 0 {
-			query.Offset = offset
-		}
-	}
-
-	if startTime := r.URL.Query().Get("start_time"); startTime != "" {
-		if t, err := time.Parse(time.RFC3339, startTime); err == nil {
-			query.StartTime = t
-		}
-	}
-
-	if endTime := r.URL.Query().Get("end_time"); endTime != "" {
-		if t, err := time.Parse(time.RFC3339, endTime); err == nil {
-			query.EndTime = t
-		}
-	}
-
-	if service := r.URL.Query().Get("service"); service != "" {
-		query.Service = strings.TrimSpace(service)
-	}
-
-	if tool := r.URL.Query().Get("tool"); tool != "" {
-		query.Tool = strings.TrimSpace(tool)
-	}
-
-	if logLevel := r.URL.Query().Get("log_level"); logLevel != "" {
-		query.LogLevel = strings.TrimSpace(logLevel)
-	}
-
-	if entryType := r.URL.Query().Get("type"); entryType != "" {
-		query.Type = strings.TrimSpace(entryType)
-	}
-
-	if orderBy := r.URL.Query().Get("order_by"); orderBy != "" {
-		validFields := []string{"timestamp", "service", "tool", "type", "log_level"}
-		if slices.Contains(validFields, orderBy) {
-			query.OrderBy = orderBy
-		}
-	}
-
-	if orderDesc := r.URL.Query().Get("order_desc"); orderDesc != "" {
-		query.OrderDesc = orderDesc == "true"
-	}
-
-	return query
-}

web_service_test.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+	"watch-tool/models"
+)
+
+type MockStorage struct {
+}
+
+func (m *MockStorage) Query(ctx context.Context, q StorageQuery) ([]models.LogMessage, error) {
+	return []models.LogMessage{
+		{
+			Service: "tixstream",
+			Fields: map[string]any{
+				"transfer_id": "uuid-1234",
+				"direction":   "incoming",
+			},
+		},
+		{
+			Service: "tixstream",
+			Fields: map[string]any{
+				"transfer_id": "uuid-1234",
+				"direction":   "incoming",
+			},
+		},
+		{
+			Service: "tixstream",
+			Fields: map[string]any{
+				"transfer_id": "uuid-5678",
+				"direction":   "outgoing",
+			},
+		},
+		{
+			Service: "tjm",
+			Fields: map[string]any{
+				"transfer_name_raw": "20250927-ABC-test-in",
+				"correlation_id":    "corr-9999",
+			},
+		},
+	}, nil
+}
+
+func (m *MockStorage) Store(ctx context.Context, entries *models.LogMessage) error { return nil }
+func (m *MockStorage) Close() error                                                { return nil }
+func (m *MockStorage) GetUnexportedEntries(ctx context.Context, limit int) ([]models.LogMessage, error) {
+	return nil, nil
+}
+func (m *MockStorage) MarkAsExported(ctx context.Context, entries []models.LogMessage) error {
+	return nil
+}
+func (m *MockStorage) DeleteOldEntries(ctx context.Context, cutoff time.Time) (int64, error) {
+	return 0, nil
+}
+func (m *MockStorage) GetStats(ctx context.Context) (map[string]any, error)              { return nil, nil }
+func (m *MockStorage) StoreBatch(ctx context.Context, entries []models.LogMessage) error { return nil }
+
+func TestWebService_HandleServiceStats(t *testing.T) {
+	mockStorage := &MockStorage{}
+	cfg := &Config{WebService: WebConfig{Enabled: true}}
+	ws := NewWebService(cfg, mockStorage)
+
+	req, err := http.NewRequest("GET", "/api/service/tixstream/stats", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	req.SetPathValue("service", "tixstream")
+
+	rr := httptest.NewRecorder()
+
+	ws.handleServiceStats(rr, req)
+
+	if rr.Code != http.StatusOK {
+		t.Errorf("handler returned wrong status code: got %v want %v", rr.Code, http.StatusOK)
+	}
+
+	var response map[string]any
+	if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
+		t.Fatal(err)
+	}
+
+	counts := response["transfer_counts"].(map[string]any)
+	total := int(counts["total"].(float64))
+	incoming := int(counts["incoming"].(float64))
+	outgoing := int(counts["outgoing"].(float64))
+
+	if total != 3 {
+		t.Errorf("Expected 3 total transfers, got %d", total)
+	}
+	if incoming != 2 {
+		t.Errorf("Expected 2 incoming transfers, got %d", incoming)
+	}
+	if outgoing != 1 {
+		t.Errorf("Expected 1 outgoing transfer, got %d", outgoing)
+	}
+}