feat: implement drain3 based generic log-parser

parser/factory.go

@@ -1,10 +1,20 @@
 package parser
 
-func New(serviceName, logType, hostname string) (Parser, error) {
-	switch logType {
+import "codeberg.org/pata1704/drain3"
+
+type ParserConfig struct {
+	ServiceName string
+	LogType     string
+	Hostname    string
+	DrainConfig *drain3.Config
+	StateDir    string
+}
+
+func New(cfg ParserConfig) (Parser, error) {
+	switch cfg.LogType {
 	case "json":
 		return &JSONParser{}, nil
 	default:
-		return NewGenericParser(serviceName, hostname), nil
+		return NewGenericParser(cfg.ServiceName, cfg.Hostname, cfg.DrainConfig, cfg.StateDir), nil
 	}
 }

parser/generic_parser.go

@@ -3,21 +3,28 @@ package parser
 import (
 	"fmt"
 	"log/slog"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 	"watch-tool/models"
 	"watch-tool/patterns"
+
+	"codeberg.org/pata1704/drain3"
 )
 
 type GenericParser struct {
-	ServiceName string
-	Hostname    string
-	Extractors  []patterns.CompiledExtractor
-	CommonExt   []patterns.CompiledExtractor
+	ServiceName    string
+	Hostname       string
+	Extractors     []patterns.CompiledExtractor
+	CommonExt      []patterns.CompiledExtractor
+	drainMiner     *drain3.TemplateMiner
+	drainEnabled   bool
+	drainStatePath string
 }
 
-func NewGenericParser(serviceName, hostname string) *GenericParser {
+func NewGenericParser(serviceName, hostname string, drainCfg *drain3.Config, stateDir string) *GenericParser {
 	repo := patterns.GetInstance()
 
 	var svcExt, commonExt []patterns.CompiledExtractor
@@ -28,12 +35,36 @@ func NewGenericParser(serviceName, hostname string) *GenericParser {
 		slog.Error("CRITICAL: Pattern Repository is nil. Parser will not work correctly.")
 	}
 
-	return &GenericParser{
+	parser := &GenericParser{
 		ServiceName: serviceName,
 		Hostname:    hostname,
 		Extractors:  svcExt,
 		CommonExt:   commonExt,
 	}
+	if drainCfg != nil && stateDir != "" {
+		parser.drainEnabled = true
+
+		parser.drainStatePath = filepath.Join(stateDir, serviceName+".bin")
+
+		if err := os.MkdirAll(stateDir, 0755); err != nil {
+			slog.Error("Failed to create drain3 state dir", "error", err)
+			parser.drainEnabled = false
+			return parser
+		}
+
+		persister := drain3.NewFilePersistence(parser.drainStatePath, false)
+
+		state, err := persister.LoadState()
+		if err == nil && state != nil {
+			parser.drainMiner = drain3.NewTemplateMiner(drainCfg, persister)
+			slog.Info("Drain3 state loaded", "service", serviceName, "clusters", len(state.Clusters))
+		} else {
+			parser.drainMiner = drain3.NewTemplateMiner(drainCfg, persister)
+			slog.Info("Drain3 initialized fresh", "service", serviceName)
+		}
+	}
+
+	return parser
 }
 
 func (p *GenericParser) Parse(line string) (models.LogMessage, error) {
@@ -51,6 +82,15 @@ func (p *GenericParser) Parse(line string) (models.LogMessage, error) {
 		return entry, nil
 	}
 
+	if p.drainEnabled && p.drainMiner != nil {
+		cluster := p.drainMiner.AddLogMessage(trimmedLine)
+		if cluster != nil {
+			entry.Fields["drain_template_id"] = cluster.ClusterID
+			entry.Fields["drain_template"] = cluster.TemplateMined
+			// Optional: Parameter extrahieren, die Drain gefunden hat (Wildcards)
+		}
+	}
+
 	allExtractors := append(p.CommonExt, p.Extractors...)
 
 	matchedAny := false
@@ -92,6 +132,17 @@ func (p *GenericParser) Parse(line string) (models.LogMessage, error) {
 	return entry, nil
 }
 
+func (p *GenericParser) Close() error {
+	if p.drainEnabled && p.drainMiner != nil {
+		if err := p.drainMiner.SaveState("shutdown"); err != nil {
+			slog.Error("Failed to save drain3 state", "service", p.ServiceName, "error", err)
+			return err
+		}
+		slog.Debug("Drain3 state saved", "service", p.ServiceName)
+	}
+	return nil
+}
+
 func (p *GenericParser) safeConvert(value, typeDef string) any {
 	if value == "" || value == "-" {
 		if strings.HasPrefix(typeDef, "int") || strings.HasPrefix(typeDef, "float") {

parser/generic_parser_test.go

@@ -0,0 +1,198 @@
+package parser
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+	"watch-tool/patterns"
+
+	"codeberg.org/pata1704/drain3"
+)
+
+func setupPatterns(t *testing.T) {
+	content := `
+patterns:
+  common:
+    extractors:
+      - name: "syslog_header"
+        regex: '^\w{3} \d{2} \d{2}:\d{2}:\d{2} (?P<hostname>\S+) .*'
+        fields:
+          hostname: "string"
+
+  test_service:
+    extractors:
+      - name: "data_line"
+        regex: 'Data: id=(?P<id>\d+) size=(?P<size_mb>[0-9.]+) active=(?P<is_active>true|false) empty=(?P<empty_val>\S+)'
+        fields:
+          id: "int"
+          size_mb: "float"
+          is_active: "bool"
+          empty_val: "int" # Testet Fallback bei "-"
+`
+	tmpfile, err := os.CreateTemp("", "patterns_parser_test_*.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(content)); err != nil {
+		t.Fatal(err)
+	}
+	tmpfile.Close()
+
+	if err := patterns.GetInstance().Load(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGenericParser_Parse_Regex(t *testing.T) {
+	setupPatterns(t)
+
+	p := NewGenericParser("test_service", "localhost", nil, "")
+
+	line := "Sep 28 10:00:00 myhost Data: id=42 size=12.5 active=true empty=-"
+	entry, err := p.Parse(line)
+	if err != nil {
+		t.Fatalf("Parse failed: %v", err)
+	}
+
+	if entry.Host != "myhost" {
+		t.Errorf("Expected host 'myhost', got '%s'", entry.Host)
+	}
+
+	if val, ok := entry.Fields["id"].(int); !ok || val != 42 {
+		t.Errorf("Expected id=42 (int), got %v (%T)", entry.Fields["id"], entry.Fields["id"])
+	}
+
+	if val, ok := entry.Fields["size_mb"].(float64); !ok || val != 12.5 {
+		t.Errorf("Expected size_mb=12.5 (float), got %v", entry.Fields["size_mb"])
+	}
+
+	if val, ok := entry.Fields["is_active"].(bool); !ok || val != true {
+		t.Errorf("Expected is_active=true, got %v", entry.Fields["is_active"])
+	}
+
+	if val, ok := entry.Fields["empty_val"].(int); !ok || val != 0 {
+		t.Errorf("Expected empty_val=0 for '-', got %v", entry.Fields["empty_val"])
+	}
+}
+
+func TestGenericParser_Drain3_Integration(t *testing.T) {
+	setupPatterns(t)
+
+	opts := &slog.HandlerOptions{Level: slog.LevelDebug}
+	logger := slog.New(slog.NewTextHandler(os.Stdout, opts))
+	slog.SetDefault(logger)
+
+	tmpDir, err := os.MkdirTemp("", "drain_state_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	drainCfg := &drain3.Config{
+		Depth:       4,
+		SimTh:       0.5,
+		MaxChildren: 100,
+		MaxClusters: 100,
+	}
+
+	serviceName := "test_service"
+	p := NewGenericParser(serviceName, "localhost", drainCfg, tmpDir)
+
+	log1 := "User admin logged in from 192.168.1.1"
+	log2 := "User guest logged in from 10.0.0.1"
+
+	entry1, _ := p.Parse(log1)
+	if entry1.Fields["drain_template_id"] == nil {
+		t.Error("Drain3 did not assign a template ID for log1")
+	}
+
+	entry2, _ := p.Parse(log2)
+
+	id1 := entry1.Fields["drain_template_id"]
+	id2 := entry2.Fields["drain_template_id"]
+	t.Logf("IDs: %v -> %v", id1, id2)
+	t.Logf("Template 2: %s", entry2.Fields["drain_template"])
+
+	if err := p.Close(); err != nil {
+		t.Fatalf("Close failed: %v", err)
+	}
+
+	expectedFile := filepath.Join(tmpDir, serviceName+".bin")
+
+	if info, err := os.Stat(expectedFile); os.IsNotExist(err) {
+		t.Errorf("Drain3 state file NOT found at: %s", expectedFile)
+
+		entries, _ := os.ReadDir(tmpDir)
+		t.Logf("Listing directory %s:", tmpDir)
+		for _, e := range entries {
+			t.Logf(" - Found file: %s", e.Name())
+		}
+	} else {
+		t.Logf("Success: State file found (%d bytes)", info.Size())
+	}
+}
+
+func TestGenericParser_Robustness(t *testing.T) {
+	setupPatterns(t)
+
+	p := NewGenericParser("test_service", "localhost", nil, "")
+
+	tests := []struct {
+		name          string
+		log           string
+		checkField    string
+		expectedValue any
+		shouldFail    bool
+	}{
+		{
+			name:          "Empty Line",
+			log:           "",
+			checkField:    "",
+			expectedValue: nil,
+			shouldFail:    false,
+		},
+		{
+			name:          "Type Mismatch Int (Text instead of Int)",
+			log:           "Data: id=abc size=12.5 active=true empty=-",
+			checkField:    "_parse_status",
+			expectedValue: "failed",
+		},
+		{
+			name:          "Value Missing (Dash) for Int",
+			log:           "Data: id=1 size=1.0 active=true empty=-",
+			checkField:    "empty_val",
+			expectedValue: 0,
+		},
+		{
+			name:          "Value Missing (Dash) for Float",
+			log:           "Data: id=1 size=1.0 active=true empty=0",
+			checkField:    "size_mb",
+			expectedValue: 1.0,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			entry, err := p.Parse(tc.log)
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if tc.checkField != "" {
+				val, exists := entry.Fields[tc.checkField]
+				if tc.expectedValue == "failed" {
+					if !exists || val != "failed" {
+						t.Errorf("Expected parse failure status, got %v", val)
+					}
+				} else {
+					if val != tc.expectedValue {
+						t.Errorf("Field %s: expected %v, got %v", tc.checkField, tc.expectedValue, val)
+					}
+				}
+			}
+		})
+	}
+}

parser/json_parser.go

@@ -17,3 +17,7 @@ func (j *JSONParser) Parse(line string) (models.LogMessage, error) {
 	}
 	return logMsg, nil
 }
+
+func (p *JSONParser) Close() error {
+	return nil
+}

parser/parser.go

@@ -6,4 +6,5 @@ import (
 
 type Parser interface {
 	Parse(line string) (models.LogMessage, error)
+	Close() error
 }