refactor: clean up and add example config and pattern

build.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-set -e
-PACKAGE_DIR="./packages"
-PACKAGE_NAME="./tixel-watch"
-
-if [ -d "${PACKAGE_DIR}" ]; then
-  rm -rf "${PACKAGE_DIR:?}/"*
-else
-  mkdir -p "${PACKAGE_DIR}"
-fi
-
-if [ -d "${PACKAGE_NAME}" ]; then
-  rm -rf "${PACKAGE_NAME:?}"
-fi
-
-CGO_ENABLED=0 go build -ldflags="-s -w" -o "$PACKAGE_DIR"/tixel-watch -buildvcs .
-
-cp -r ./tixel-watch.service $PACKAGE_DIR/
-cp -r ./configs/ $PACKAGE_DIR/
-cp -r ./install.sh $PACKAGE_DIR/
-mv $PACKAGE_DIR tixel-watch
-
-tar -czvf tixel-watch.tar.gz ./tixel-watch

config.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"log/slog"
+	"os"
 	"regexp"
 	"time"
 
@@ -156,7 +157,7 @@ func setConfigDefaults() {
 	viper.SetDefault("export.retry_backoff", "5s")
 	viper.SetDefault("export.health_check_interval", "60s")
 	viper.SetDefault("localstorage.enabled", true)
-	viper.SetDefault("localstorage.db_path", "./tixel_watch.db")
+	viper.SetDefault("localstorage.db_path", "./watch.db")
 	viper.SetDefault("localstorage.rotation.max_size_bytes", int64(100*1024*1024))
 	viper.SetDefault("localstorage.rotation.max_age_hours", 24)
 	viper.SetDefault("localstorage.rotation.max_files", 7)
@@ -171,9 +172,13 @@ func setConfigDefaults() {
 }
 
 func LoadConfig() (*Config, error) {
+	home, err := os.UserConfigDir()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get user config dir: %w", err)
+	}
 	viper.SetConfigName("config")
 	viper.AddConfigPath(".")
-	viper.AddConfigPath("/opt/tixel/tixel-watch/")
+	viper.AddConfigPath(home)
 	viper.SetConfigType("yaml")
 
 	setConfigDefaults()

configs/elasticsearch.yml

@@ -1,119 +0,0 @@
-# ======================== Elasticsearch Configuration =========================
-#
-# NOTE: Elasticsearch comes with reasonable defaults for most settings.
-#       Before you set out to tweak and tune the configuration, make sure you
-#       understand what are you trying to accomplish and the consequences.
-#
-# The primary way of configuring a node is via this file. This template lists
-# the most important settings you may want to configure for a production cluster.
-#
-# Please consult the documentation for further information on configuration options:
-# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
-#
-# ---------------------------------- Cluster -----------------------------------
-#
-# Use a descriptive name for your cluster:
-#
-cluster.name: tixel-elastic
-#
-# ------------------------------------ Node ------------------------------------
-#
-# Use a descriptive name for the node:
-#
-#node.name: node-1
-#
-# Add custom attributes to the node:
-#
-#node.attr.rack: r1
-#
-# ----------------------------------- Paths ------------------------------------
-#
-# Path to directory where to store the data (separate multiple locations by comma):
-#
-path.data: /var/lib/elasticsearch
-#
-# Path to log files:
-#
-path.logs: /var/log/elasticsearch
-#
-# ----------------------------------- Memory -----------------------------------
-#
-# Lock the memory on startup:
-#
-#bootstrap.memory_lock: true
-#
-# Make sure that the heap size is set to about half the memory available
-# on the system and that the owner of the process is allowed to use this
-# limit.
-#
-# Elasticsearch performs poorly when the system is swapping the memory.
-#
-# ---------------------------------- Network -----------------------------------
-#
-# By default Elasticsearch is only accessible on localhost. Set a different
-# address here to expose this node on the network:
-#
-network.host: 0.0.0.0
-#
-# By default Elasticsearch listens for HTTP traffic on the first free port it
-# finds starting at 9200. Set a specific HTTP port here:
-#
-#http.port: 9200
-#
-# For more information, consult the network module documentation.
-#
-# --------------------------------- Discovery ----------------------------------
-#
-# Pass an initial list of hosts to perform discovery when this node is started:
-# The default list of hosts is ["127.0.0.1", "[::1]"]
-#
-#discovery.seed_hosts: ["host1", "host2"]
-#
-# Bootstrap the cluster using an initial set of master-eligible nodes:
-#
-#cluster.initial_master_nodes: ["node-1", "node-2"]
-#
-# For more information, consult the discovery and cluster formation module documentation.
-#
-# ---------------------------------- Various -----------------------------------
-#
-# Allow wildcard deletion of indices:
-#
-#action.destructive_requires_name: false
-
-#----------------------- BEGIN SECURITY AUTO CONFIGURATION -----------------------
-#
-# The following settings, TLS certificates, and keys have been automatically      
-# generated to configure Elasticsearch security features on 26-08-2025 14:51:23
-#
-# --------------------------------------------------------------------------------
-
-# Enable security features
-xpack.security.enabled: false
-
-xpack.security.enrollment.enabled: false
-
-# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents
-xpack.security.http.ssl:
-  enabled: false
-  keystore.path: certs/http.p12
-
-# Enable encryption and mutual authentication between cluster nodes
-xpack.security.transport.ssl:
-  enabled: false
-  verification_mode: certificate
-  keystore.path: certs/transport.p12
-  truststore.path: certs/transport.p12
-# Create a new cluster with the current node only
-# Additional nodes can still join the cluster later
-cluster.initial_master_nodes: ["frankfurt.tixeltec.de"]
-
-# Allow HTTP API connections from anywhere
-# Connections are encrypted and require user authentication
-http.host: 0.0.0.0
-
-# Allow other nodes to join the cluster from anywhere
-# Connections are encrypted and mutually authenticated
-transport.host: 0.0.0.0
-
-#----------------------- END SECURITY AUTO CONFIGURATION -------------------------

configs/example-config.yaml

@@ -1,35 +1,74 @@
+
+export:
+  enabled: true
+  batch_size: 100
+  export_interval: "30s"
+  retry_attempts: 5
+  retry_backoff: "10s"
+  health_check_interval: "60s"
+
+localstorage:
+  enabled: true
+  db_path: "./watch.db"
+  rotation:
+    max_sizes_bytes: 100 * 1024 * 1024
+    max_age_hours: 24
+    max_files: 3
+    check_interval_minuntes: 5
+    archive_dir: ""
+
 elasticsearch:
-  url: "http://localhost:9200"
+  enabled: true
-  index: "tixel"
+  url: "http://10.0.0.99:9200"
-  username: "elastic"
+  index: "watch"
-  password: "79QQ4JGTa3R_nkqA=MxW"
+  username: "your-configured-user"
+  password: "your-super-secret-password"
+  api_key: "your-api-key"
   timeout: 30
 
 web_service:
   enabled: true
-  host: "localhost"
+  host: "0.0.0.0"
-  port: 8080
+  port: 9090
 
 system_metrics:
   enabled: true
   collect_cpu: true
   collect_memory: true
   collect_disk: true
-  collect_network: false
+  collect_network: true
   disk_paths:
     - "/"
     - "/var"
     - "/home"
   network_interfaces:
-    - "eth0"
+    - "ens6"
-    - "wlan0"
+  collect_network_connections: true
+  collect_load_average: true
+  collect_tcp_stats: true
+  collect_filehandles: true
+  collect_disk_io: true
+  collect_network_latency: true
+  collect_bandwidth_usage: true
+  transfer_ports: 60003
+  latency_test_hosts: "www.google.de"
 
 poll_interval_seconds: 30
+patterns_file: "./configs/patterns.yaml"
 
 logging:
   level: "info"
   file_path: "/var/log/system-monitor.log"
 
+drain3:
+  enabled: true
+  state_dir: "./drain3_states"
+  depth: 4
+  sim_th: 0.4
+  max_children: 100
+  max_clusters: 1000
+  save_interval: 60
+
 services:
   - name: "nginx"
     service: "nginx.service"
@@ -37,18 +76,6 @@ services:
     since_time: ""
     priority: "info"
 
-  - name: "tixstream"
-    service: "tixstream.service"
-    enabled: true
-    since_time: ""
-    priority: "debug"
-
-  - name: "transfer-job-manager"
-    service: "transfer-job-manager.service"
-    enabled: true
-    since_time: ""
-    priority: "debug"
-
 tools:
   - name: "nginx-access"
     log_file: "/var/log/nginx/access.log"
@@ -82,9 +109,3 @@ tools:
         tid: "thread_id"
         message: "error_message"
 
-  - name: "nginx-tjm"
-    log_file: "/var/log/nginx/access_tjm.log"
-    enabled: true
-    buffer_size: 100
-    format:
-      name: "custom"

configs/example-patterns.yaml

@@ -0,0 +1,36 @@
+patterns:
+  common:
+    extractors:
+      - name: "syslog_header"
+        regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+) (?P<process_info>[^:]+):\s*(?P<message_rest>.*)$'
+        fields:
+          syslog_timestamp: "time:Jan 02 15:04:05"
+          hostname: "string"
+          process_info: "string"
+          message_rest: "string"
+      
+      - name: "iso8601_timestamp"
+        regex: '(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)'
+        fields:
+          timestamp: "time:2006-01-02T15:04:05.000000Z"
+
+  nginx:
+    extractors:
+      - name: "access_log"
+        regex: '^(?P<client_ip>\S+)\s+\S+\s+(?P<remote_user>\S+)\s+\[(?P<timestamp_nginx>[^\]]+)\]\s+"(?P<request>[^"]+)"\s+(?P<status_code>\d+)\s+(?P<bytes_sent>\d+|-)'
+        fields:
+          client_ip: "string"
+          remote_user: "string"
+          timestamp_nginx: "string"
+          request: "string"
+          status_code: "int"
+          bytes_sent: "int"
+
+  my-app:
+    extractors:
+      - name: "app_log"
+        regex: '^\[(?P<level>\w+)\] id=(?P<request_id>\d+) duration=(?P<duration_ms>\d+)ms'
+        fields:
+          level: "string"
+          request_id: "int"
+          duration_ms: "int"

configs/patterns.yml

@@ -1,164 +0,0 @@
-patterns:
-  # ===========================================================================
-  # Common / Shared Patterns
-  # ===========================================================================
-  common:
-    extractors:
-      - name: "syslog_header"
-        regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+) (?P<process_info>[^:]+):\s*(?P<message_rest>.*)$'
-        fields:
-          syslog_timestamp: "time:Jan 02 15:04:05"
-          hostname: "string"
-          process_info: "string"
-          message_rest: "string"
-          
-      - name: "timestamp_rfc3339"
-        regex: '(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)'
-        fields:
-          timestamp: "time:2006-01-02T15:04:05.000000Z"
-
-  # ===========================================================================
-  # TIXstream Service
-  # Deckt ab: tsServicePattern, tsTransferIDPattern, tsDetailPattern1-4
-  # ===========================================================================
-  tixstream:
-    extractors:
-      - name: "service_log_base"
-        regex: '^(?P<log_level>\S+)\s+(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?P<message>.*)'
-        fields:
-          log_level: "string"
-          timestamp: "time:2006-01-02 15:04:05.000000"
-          message: "string"
-
-      - name: "transfer_id_extraction"
-        regex: '^(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?P<message>.*)'
-        fields:
-          transfer_id: "string"
-          message: "string"
-
-      - name: "transfer_start_in"
-        regex: 'in: Transfer start (?P<thread_info>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<file_count>\d+) size=(?P<size_mb>[0-9.]+) MByte chunksize=(?P<chunk_size>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_rate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<destination>\S+) sender-id=(?P<sender_id>\S+)'
-        fields:
-          thread_info: "string" # z.B. "1/4" - Typisierung hier schwierig, also String
-          buffers: "int"
-          file_count: "int"
-          size_mb: "float"
-          chunk_size: "int"
-          streams: "int"
-          target_rate: "float"
-          protocol: "string"
-          destination: "string"
-          sender_id: "string"
-          direction: "string" # Wir können statische Felder im Parser injecten oder hier als "implizit" betrachten
-
-      - name: "transfer_start_remote_out"
-        regex: 'out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s'
-        fields:
-          target: "string"
-          duration: "float"
-
-      - name: "transfer_start_out"
-        regex: 'out: Transfer start (?P<thread_info>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<file_count>\d+) size=(?P<size_mb>[0-9.]+) MByte chunksize=(?P<chunk_size>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_rate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<source>\S+) receiver=(?P<receiver>\S+)'
-        fields:
-          thread_info: "string"
-          buffers: "int"
-          file_count: "int"
-          size_mb: "float"
-          chunk_size: "int"
-          streams: "int"
-          target_rate: "float"
-          protocol: "string"
-          source: "string"
-          receiver: "string"
-
-      - name: "transfer_start_generic"
-        regex: 'out: Start transfer (?P<thread_info>\d+/\d+), src=(?P<source>[^ ]*) dest=(?P<destination>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)'
-        fields:
-          thread_info: "string"
-          source: "string"
-          destination: "string"
-          item0: "string"
-          count: "int"
-
-  # ===========================================================================
-  # Transfer Job Manager (TJM)
-  # Deckt ab: tjmServicePattern, tjmTransferNamePattern, tjmTransferIDPattern1/2
-  # ===========================================================================
-  transfer-job-manager:
-    extractors:
-      - name: "service_log_base"
-        regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?P<log_level>\S+)\s+(?P<pid>\d+).*?\[(?P<correlation_id>[^\]]*)\]\s+\[(?P<username>[^\]]*)\]\s+\[(?P<thread_id>[^\]]*)\]\s+(?P<java_class>.*?)\s+:\s+(?P<message>.*)'
-        fields:
-          timestamp: "time:2006-01-02 15:04:05.000"
-          log_level: "string"
-          pid: "int"
-          correlation_id: "string"
-          username: "string"
-          thread_id: "string"
-          java_class: "string"
-          message: "string"
-
-      - name: "transfer_name_info"
-        regex: '^(?P<transfer_name_raw>\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (?P<message>.*)$'
-        fields:
-          transfer_name_raw: "string"
-          message: "string"
-
-      - name: "transfer_id_mid"
-        regex: '(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)'
-        fields:
-          transfer_id: "string"
-          message: "string"
-
-      - name: "transfer_id_prefixed"
-        regex: '(?P<prefix>.*)(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)'
-        fields:
-          prefix: "string"
-          transfer_id: "string"
-          message: "string"
-
-  # ===========================================================================
-  # Access Manager & TCC
-  # Deckt ab: amServicePattern, tccServicePattern
-  # ===========================================================================
-  access-manager:
-    extractors:
-      - name: "spring_boot_log"
-        regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P<log_level>\w+)\s+(?P<pid>\d+)\s+---\s+\[\s*(?P<thread_id>[^\]]*)\]\s+(?P<logger>[\w\.]+)\s*:\s+(?P<message>.*)$'
-        fields:
-          timestamp: "time:2006-01-02T15:04:05.000000Z"
-          log_level: "string"
-          pid: "int"
-          thread_id: "string"
-          logger: "string"
-          message: "string"
-
-  tixel-control-center:
-    extractors:
-      - name: "spring_boot_log"
-        regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P<log_level>\w+)\s+(?P<pid>\d+)\s+---\s+\[\s*(?P<thread_id>[^\]]*)\]\s+(?P<logger>[\w\.]+)\s*:\s+(?P<message>.*)$'
-        fields:
-          timestamp: "time:2006-01-02T15:04:05.000000Z"
-          log_level: "string"
-          pid: "int"
-          thread_id: "string"
-          logger: "string"
-          message: "string"
-
-  # ===========================================================================
-  # Nginx
-  # Deckt ab: nginxAccessPattern
-  # ===========================================================================
-  nginx:
-    extractors:
-      - name: "access_log"
-        regex: '^(?P<client_ip>\S+)\s+\S+\s+(?P<remote_user>\S+)\s+\[(?P<timestamp_nginx>[^\]]+)\]\s+"(?P<request>[^"]+)"\s+(?P<status_code>\d+)\s+(?P<bytes_sent>\d+|-)\s*(?:"(?P<referer>[^"]*)"\s+"(?P<user_agent>[^"]*)")?'
-        fields:
-          client_ip: "string"
-          remote_user: "string"
-          timestamp_nginx: "string" 
-          request: "string"
-          status_code: "int"
-          bytes_sent: "int"
-          referer: "string"
-          user_agent: "string"

elasticsearch.go

@@ -70,7 +70,7 @@ func (esc *ElasticsearchClient) SendBatch(baseIndex string, entries []models.Log
 
 	var body strings.Builder
 	for _, entry := range entries {
-		indexName := "tixel"
+		indexName := "watch"
 
 		indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName)
 		body.WriteString(indexLine)
@@ -117,7 +117,7 @@ func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics mode
 		return fmt.Errorf("JSON marshalling error: %w", err)
 	}
 
-	systemIndex := "tixel"
+	systemIndex := "watch"
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()

install.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-set -e
-
-ES_VERSION="9.1.2"
-ES_DEB_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-amd64.deb"
-ES_RPM_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-x86_64.rpm"
-ES_CONFIG_DIR="/etc/elasticsearch"
-ES_JVM_OPTIONS="/etc/elasticsearch/jvm.options"
-ES_JVM_OPTIONS_D="/etc/elasticsearch/jvm.options.d"
-GO_SERVICE_NAME="tixel-watch"
-GO_INSTALL_TARGET="/opt/tixel/tixel-watch"
-
-install_es_deb() {
-  echo "Installing Elasticsearch (Debian package)..."
-  wget "${ES_DEB_URL}" -O elasticsearch.deb
-  sudo dpkg -i elasticsearch.deb
-  sudo apt-get install -f -y
-}
-
-install_es_rpm() {
-  echo "Installing Elasticsearch (RPM package)..."
-  wget "${ES_RPM_URL}" -O elasticsearch.rpm
-  sudo rpm --install elasticsearch.rpm
-}
-
-setup_configuration() {
-  echo "Copying Elasticsearch configuration files..."
-  sudo cp ./configs/elasticsearch.yml "${ES_CONFIG_DIR}/elasticsearch.yml"
-  sudo cp ./configs/jvm.options "${ES_JVM_OPTIONS}"
-  sudo cp -r ./configs/jvm.options.d "${ES_JVM_OPTIONS_D}"
-  sudo chown root:elasticsearch "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}"
-  sudo chmod 640 "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}"
-}
-
-setup_tixel_watch_service() {
-  echo "Setting up tixel-watch systemd service..."
-  if [ ! -d ${GO_INSTALL_TARGET} ]; then
-    mkdir -p ${GO_INSTALL_TARGET}
-  fi
-  sudo cp ./tixel-watch "$GO_INSTALL_TARGET"/
-  sudo cp ./configs/config.yaml "$GO_INSTALL_TARGET"/
-  sudo cp ./${GO_SERVICE_NAME}.service /etc/systemd/system/${GO_SERVICE_NAME}.service
-  sudo systemctl daemon-reload
-  sudo systemctl enable "${GO_SERVICE_NAME}"
-}
-
-start_services() {
-  echo "Enabling and starting Elasticsearch service..."
-  sudo systemctl enable elasticsearch
-  sudo systemctl start elasticsearch
-  echo "Starting tixel-watch service..."
-  sudo systemctl start "${GO_SERVICE_NAME}"
-}
-
-main() {
-  if command -v apt-get &>/dev/null; then
-    install_es_deb
-  elif command -v yum &>/dev/null || command -v dnf &>/dev/null; then
-    install_es_rpm
-  else
-    echo "Unsupported package manager. Aborting."
-    exit 1
-  fi
-
-  setup_configuration
-  setup_tixel_watch_service
-  start_services
-  echo "All done."
-}
-
-main "$@"

tixel-watch.service

@@ -1,19 +0,0 @@
-[Unit]
-Description=tixel-watch
-After=network.target
-
-[Service]
-Type=simple
-User=tixstream
-Group=tixstream
-WorkingDirectory=/opt/tixel/tixel-watch
-ExecStart=/opt/tixel/tixel-watch/tixel-watch
-Restart=on-failure
-RestartSec=5
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=tixel-watch
-
-[Install]
-WantedBy=multi-user.target
-

web_service.go

@@ -98,7 +98,7 @@ func (ws *WebService) handleHealth(w http.ResponseWriter, r *http.Request) {
 		status["storage_status"] = "healthy"
 	}
 	statusMap := make(map[string]any)
-	statusMap["tixel-watch"] = status
+	statusMap["watch"] = status
 
 	for _, service := range ws.config.Services {
 		statusCommand := []string{"sudo", "systemctl", "status", service.Name, "--no-pager"}
@@ -160,7 +160,7 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	filename := fmt.Sprintf("tixel_export_%s.json", time.Now().Format("20060102_150405"))
+	filename := fmt.Sprintf("watch_export_%s.json", time.Now().Format("20060102_150405"))
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
 
@@ -169,7 +169,7 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
 			"timestamp":   time.Now(),
 			"entry_count": len(entries),
 			"query":       query,
-			"exported_by": "tixel-watch",
+			"exported_by": "watch",
 		},
 		"entries": entries,
 	}