commit for version used in evaluation of thesis
This commit is contained in:
commit
72635dc7b9
27 changed files with 6084 additions and 0 deletions
294
cmd/pipeline/main.go
Normal file
294
cmd/pipeline/main.go
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
// Command pipeline is the entry point for the MFT anomaly detection pipeline.
|
||||
//
|
||||
// Startup order:
|
||||
// 1. Load and compile config (masking patterns → *regexp.Regexp).
|
||||
// 2. Allocate channels with fixed capacities to enable backpressure.
|
||||
// 3. Start HealthMonitor.
|
||||
// 4. Start collectors (MetricCollector, LogCollector).
|
||||
// 5. Start TransformEngine (DuckDB, schema, pre-compiled query).
|
||||
// 6. Start DetectionLayer.
|
||||
// 7. Start anomaly sink goroutine.
|
||||
// 8. Wait for SIGINT / SIGTERM.
|
||||
// 9. Graceful shutdown in reverse order.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"codeberg.org/pata1704/guenther/internal/collector"
|
||||
"codeberg.org/pata1704/guenther/internal/config"
|
||||
"codeberg.org/pata1704/guenther/internal/detect"
|
||||
"codeberg.org/pata1704/guenther/internal/health"
|
||||
"codeberg.org/pata1704/guenther/internal/transform"
|
||||
"codeberg.org/pata1704/guenther/pkg/types"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfgPath := flag.String("config", "configs/default.yaml", "path to config file")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.LoadConfig(*cfgPath)
|
||||
if err != nil {
|
||||
log.Fatalf("load config %q: %v", *cfgPath, err)
|
||||
}
|
||||
if err := cfg.Compile(); err != nil {
|
||||
log.Fatalf("compile masking patterns: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
logChan := make(chan types.LogEvent, 1_000)
|
||||
metricChan := make(chan types.MetricSnapshot, 100)
|
||||
serviceStatusChan := make(chan types.ServiceStatus, 100)
|
||||
featureChan := make(chan types.FeatureVector, 10)
|
||||
anomalyChan := make(chan types.AnomalyResult, 50)
|
||||
|
||||
hm := health.NewHealthMonitor()
|
||||
hm.Start(ctx, 5*time.Second)
|
||||
|
||||
metricColl := collector.NewMetricCollector(
|
||||
metricChan, hm.Chan(),
|
||||
time.Second,
|
||||
cfg.Ingestion.NetInterface,
|
||||
cfg.Ingestion.DiskDevice,
|
||||
)
|
||||
logColl := collector.NewLogCollector(cfg, logChan, hm.Chan())
|
||||
sysColl := collector.NewSystemctlCollector(
|
||||
cfg.Ingestion.SystemctlServices,
|
||||
5*time.Second,
|
||||
serviceStatusChan,
|
||||
hm.Chan(),
|
||||
)
|
||||
|
||||
metricColl.Start(ctx)
|
||||
if err := logColl.Start(ctx); err != nil {
|
||||
log.Fatalf("start log collector: %v", err)
|
||||
}
|
||||
sysColl.Start(ctx)
|
||||
|
||||
engine, err := transform.NewTransformEngine(cfg, logChan, metricChan, serviceStatusChan, featureChan, hm.Chan())
|
||||
if err != nil {
|
||||
log.Fatalf("create transform engine: %v", err)
|
||||
}
|
||||
engine.Start(ctx)
|
||||
|
||||
detector, err := buildDetector(cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("build detector: %v", err)
|
||||
}
|
||||
detLayer := detect.NewDetectionLayer(detector, featureChan, anomalyChan, hm.Chan())
|
||||
|
||||
if cfg.Detection.AutoScaling.Enabled {
|
||||
if sd, ok := detector.(*detect.SwitchableDetector); ok {
|
||||
sc := detect.NewScalingController(
|
||||
sd,
|
||||
cfg.Detection.AutoScaling.HighThreshold,
|
||||
cfg.Detection.AutoScaling.CritThreshold,
|
||||
cfg.Detection.AutoScaling.DownThreshold,
|
||||
cfg.Detection.AutoScaling.HighDuration,
|
||||
cfg.Detection.AutoScaling.CritDuration,
|
||||
cfg.Detection.AutoScaling.DownDuration,
|
||||
)
|
||||
detLayer.SetScalingController(sc)
|
||||
log.Println("detector: auto-scaling enabled")
|
||||
} else {
|
||||
log.Println("warning: auto-scaling requested but detector is not switchable (requires SEAD ensemble)")
|
||||
}
|
||||
}
|
||||
|
||||
detLayer.Start(ctx)
|
||||
|
||||
anomalyLog := openLog(cfg.Output.AnomalyLogPath, "anomaly log")
|
||||
if anomalyLog != nil {
|
||||
defer anomalyLog.Close()
|
||||
}
|
||||
anomalyWriter := maybeWriter(anomalyLog)
|
||||
|
||||
var sinkWg sync.WaitGroup
|
||||
sinkWg.Add(1)
|
||||
go func() {
|
||||
defer sinkWg.Done()
|
||||
for res := range anomalyChan {
|
||||
writeJSON(anomalyWriter, res)
|
||||
if res.IsAnomaly {
|
||||
log.Printf("[ANOMALY] time=%s score=%.4f method=%s details=%s",
|
||||
res.Timestamp.Format(time.RFC3339), res.Score, res.Method, res.Details)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Optionally log SEAD weights periodically (when using SEAD ensemble).
|
||||
if ens, ok := detector.(*detect.EnsembleDetector); ok {
|
||||
go func() {
|
||||
t := time.NewTicker(60 * time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if ws := ens.WeightSummary(); ws != "" {
|
||||
log.Printf("[SEAD weights] %s", ws)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
log.Println("pipeline started – waiting for SIGINT / SIGTERM")
|
||||
<-ctx.Done()
|
||||
log.Println("shutting down…")
|
||||
|
||||
metricColl.Wait()
|
||||
logColl.Wait()
|
||||
engine.Wait()
|
||||
|
||||
close(featureChan)
|
||||
detLayer.Wait()
|
||||
|
||||
close(anomalyChan)
|
||||
sinkWg.Wait()
|
||||
|
||||
hm.Wait()
|
||||
log.Println("pipeline stopped")
|
||||
}
|
||||
|
||||
// buildDetector constructs the configured AnomalyDetector.
|
||||
//
|
||||
// Routing:
|
||||
// 1. detector.ensemble.enabled = true → EnsembleDetector with the method
|
||||
// specified by detector.ensemble.method ("avg"|"max"|"median"|"sead").
|
||||
// 2. Otherwise fall through to detector.method ("copod"|"rrcf"|"isolation_forest").
|
||||
func buildDetector(cfg *config.Config) (detect.AnomalyDetector, error) {
|
||||
if cfg.Detection.Ensemble.Enabled {
|
||||
method := detect.EnsembleMethod(cfg.Detection.Ensemble.Method)
|
||||
if method == "" {
|
||||
method = detect.EnsembleAVG // backward-compat default
|
||||
}
|
||||
|
||||
// Map SEAD config from YAML to detect.SEADConfig.
|
||||
seadCfg := detect.SEADConfig{
|
||||
Eta: cfg.Detection.Ensemble.SEAD.Eta,
|
||||
Lambda: cfg.Detection.Ensemble.SEAD.Lambda,
|
||||
QuantileWindow: cfg.Detection.Ensemble.SEAD.QuantileWindow,
|
||||
MinDataPoints: cfg.Detection.Ensemble.SEAD.MinDataPoints,
|
||||
Contamination: cfg.Detection.Ensemble.Contamination,
|
||||
}
|
||||
// Apply defaults for zero-value fields.
|
||||
if seadCfg.Eta == 0 {
|
||||
seadCfg.Eta = 0.10
|
||||
}
|
||||
if seadCfg.QuantileWindow == 0 {
|
||||
seadCfg.QuantileWindow = 300
|
||||
}
|
||||
if seadCfg.MinDataPoints == 0 {
|
||||
seadCfg.MinDataPoints = 20
|
||||
}
|
||||
|
||||
det, err := detect.NewEnsembleDetector(
|
||||
method,
|
||||
cfg.Detection.COPOD.BufferSize,
|
||||
cfg.Detection.COPOD.Threshold,
|
||||
detect.RRCFVariantsConfig{
|
||||
Fast: detect.RRCFVariantConfig{
|
||||
NumTrees: cfg.Detection.RRCFVariants.Fast.NumTrees,
|
||||
TreeSize: cfg.Detection.RRCFVariants.Fast.TreeSize,
|
||||
ThresholdPercentile: cfg.Detection.RRCFVariants.Fast.ThresholdPercentile,
|
||||
},
|
||||
Mid: detect.RRCFVariantConfig{
|
||||
NumTrees: cfg.Detection.RRCFVariants.Mid.NumTrees,
|
||||
TreeSize: cfg.Detection.RRCFVariants.Mid.TreeSize,
|
||||
ThresholdPercentile: cfg.Detection.RRCFVariants.Mid.ThresholdPercentile,
|
||||
},
|
||||
Slow: detect.RRCFVariantConfig{
|
||||
NumTrees: cfg.Detection.RRCFVariants.Slow.NumTrees,
|
||||
TreeSize: cfg.Detection.RRCFVariants.Slow.TreeSize,
|
||||
ThresholdPercentile: cfg.Detection.RRCFVariants.Slow.ThresholdPercentile,
|
||||
},
|
||||
},
|
||||
cfg.Detection.Ensemble.Contamination,
|
||||
seadCfg,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build ensemble detector (%s): %w", method, err)
|
||||
}
|
||||
log.Printf("detector: Ensemble method=%s contamination=%.2f", method, cfg.Detection.Ensemble.Contamination)
|
||||
if method == detect.EnsembleSEAD {
|
||||
log.Printf("detector: SEAD η=%.3f λ=%.3f quantile_window=%d",
|
||||
seadCfg.Eta, seadCfg.Lambda, seadCfg.QuantileWindow)
|
||||
|
||||
// Wrap in SwitchableDetector if using SEAD (required for 3-stage scaling).
|
||||
if sead := det.SEAD(); sead != nil {
|
||||
return detect.NewSwitchableDetector(sead), nil
|
||||
}
|
||||
}
|
||||
return det, nil
|
||||
}
|
||||
|
||||
switch cfg.Detection.Method {
|
||||
case "copod":
|
||||
return detect.NewCOPODDetector(
|
||||
cfg.Detection.COPOD.BufferSize,
|
||||
cfg.Detection.COPOD.Threshold,
|
||||
)
|
||||
case "rrcf":
|
||||
return detect.NewRRCFDetector(
|
||||
cfg.Detection.RRCF.NumTrees,
|
||||
cfg.Detection.RRCF.TreeSize,
|
||||
0,
|
||||
cfg.Detection.RRCF.ThresholdPercentile,
|
||||
), nil
|
||||
default: // "isolation_forest"
|
||||
return detect.NewIsolationForestDetector(
|
||||
5_000, 100, 100, 256, 0.05, 10.0,
|
||||
), nil
|
||||
}
|
||||
}
|
||||
|
||||
func openLog(path, label string) *os.File {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
log.Printf("warning: cannot open %s %q: %v", label, path, err)
|
||||
return nil
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func maybeWriter(f *os.File) *bufio.Writer {
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
return bufio.NewWriterSize(f, 64*1024)
|
||||
}
|
||||
|
||||
func writeJSON(w *bufio.Writer, v any) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
log.Printf("marshal: %v", err)
|
||||
return
|
||||
}
|
||||
if _, err := w.Write(append(b, '\n')); err != nil {
|
||||
log.Printf("write log: %v", err)
|
||||
return
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
log.Printf("flush log: %v", err)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue