From 17723de72f9a594f1007fad972e284ae7686d705 Mon Sep 17 00:00:00 2001 From: Patryk Hegenberg Date: Sun, 18 Jan 2026 17:51:23 +0100 Subject: [PATCH] refactor: clean up and add example config and pattern --- build.sh | 23 --- config.go | 9 +- configs/elasticsearch.yml | 119 -------------- configs/{config.yaml => example-config.yaml} | 75 ++++++--- configs/example-patterns.yaml | 36 ++++ configs/patterns.yml | 164 ------------------- elasticsearch.go | 4 +- install.sh | 71 -------- tixel-watch.service | 19 --- web_service.go | 6 +- 10 files changed, 96 insertions(+), 430 deletions(-) delete mode 100755 build.sh delete mode 100644 configs/elasticsearch.yml rename configs/{config.yaml => example-config.yaml} (60%) create mode 100644 configs/example-patterns.yaml delete mode 100644 configs/patterns.yml delete mode 100755 install.sh delete mode 100644 tixel-watch.service diff --git a/build.sh b/build.sh deleted file mode 100755 index 3cebecf..0000000 --- a/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e -PACKAGE_DIR="./packages" -PACKAGE_NAME="./tixel-watch" - -if [ -d "${PACKAGE_DIR}" ]; then - rm -rf "${PACKAGE_DIR:?}/"* -else - mkdir -p "${PACKAGE_DIR}" -fi - -if [ -d "${PACKAGE_NAME}" ]; then - rm -rf "${PACKAGE_NAME:?}" -fi - -CGO_ENABLED=0 go build -ldflags="-s -w" -o "$PACKAGE_DIR"/tixel-watch -buildvcs . - -cp -r ./tixel-watch.service $PACKAGE_DIR/ -cp -r ./configs/ $PACKAGE_DIR/ -cp -r ./install.sh $PACKAGE_DIR/ -mv $PACKAGE_DIR tixel-watch - -tar -czvf tixel-watch.tar.gz ./tixel-watch diff --git a/config.go b/config.go index 249c246..8c4ade2 100644 --- a/config.go +++ b/config.go @@ -3,6 +3,7 @@ package main import ( "fmt" "log/slog" + "os" "regexp" "time" @@ -156,7 +157,7 @@ func setConfigDefaults() { viper.SetDefault("export.retry_backoff", "5s") viper.SetDefault("export.health_check_interval", "60s") viper.SetDefault("localstorage.enabled", true) - viper.SetDefault("localstorage.db_path", "./tixel_watch.db") + viper.SetDefault("localstorage.db_path", "./watch.db") viper.SetDefault("localstorage.rotation.max_size_bytes", int64(100*1024*1024)) viper.SetDefault("localstorage.rotation.max_age_hours", 24) viper.SetDefault("localstorage.rotation.max_files", 7) @@ -171,9 +172,13 @@ func setConfigDefaults() { } func LoadConfig() (*Config, error) { + home, err := os.UserConfigDir() + if err != nil { + return nil, fmt.Errorf("unable to get user config dir: %w", err) + } viper.SetConfigName("config") viper.AddConfigPath(".") - viper.AddConfigPath("/opt/tixel/tixel-watch/") + viper.AddConfigPath(home) viper.SetConfigType("yaml") setConfigDefaults() diff --git a/configs/elasticsearch.yml b/configs/elasticsearch.yml deleted file mode 100644 index fdd1f9c..0000000 --- a/configs/elasticsearch.yml +++ /dev/null @@ -1,119 +0,0 @@ -# ======================== Elasticsearch Configuration ========================= -# -# NOTE: Elasticsearch comes with reasonable defaults for most settings. -# Before you set out to tweak and tune the configuration, make sure you -# understand what are you trying to accomplish and the consequences. -# -# The primary way of configuring a node is via this file. This template lists -# the most important settings you may want to configure for a production cluster. -# -# Please consult the documentation for further information on configuration options: -# https://www.elastic.co/guide/en/elasticsearch/reference/index.html -# -# ---------------------------------- Cluster ----------------------------------- -# -# Use a descriptive name for your cluster: -# -cluster.name: tixel-elastic -# -# ------------------------------------ Node ------------------------------------ -# -# Use a descriptive name for the node: -# -#node.name: node-1 -# -# Add custom attributes to the node: -# -#node.attr.rack: r1 -# -# ----------------------------------- Paths ------------------------------------ -# -# Path to directory where to store the data (separate multiple locations by comma): -# -path.data: /var/lib/elasticsearch -# -# Path to log files: -# -path.logs: /var/log/elasticsearch -# -# ----------------------------------- Memory ----------------------------------- -# -# Lock the memory on startup: -# -#bootstrap.memory_lock: true -# -# Make sure that the heap size is set to about half the memory available -# on the system and that the owner of the process is allowed to use this -# limit. -# -# Elasticsearch performs poorly when the system is swapping the memory. -# -# ---------------------------------- Network ----------------------------------- -# -# By default Elasticsearch is only accessible on localhost. Set a different -# address here to expose this node on the network: -# -network.host: 0.0.0.0 -# -# By default Elasticsearch listens for HTTP traffic on the first free port it -# finds starting at 9200. Set a specific HTTP port here: -# -#http.port: 9200 -# -# For more information, consult the network module documentation. -# -# --------------------------------- Discovery ---------------------------------- -# -# Pass an initial list of hosts to perform discovery when this node is started: -# The default list of hosts is ["127.0.0.1", "[::1]"] -# -#discovery.seed_hosts: ["host1", "host2"] -# -# Bootstrap the cluster using an initial set of master-eligible nodes: -# -#cluster.initial_master_nodes: ["node-1", "node-2"] -# -# For more information, consult the discovery and cluster formation module documentation. -# -# ---------------------------------- Various ----------------------------------- -# -# Allow wildcard deletion of indices: -# -#action.destructive_requires_name: false - -#----------------------- BEGIN SECURITY AUTO CONFIGURATION ----------------------- -# -# The following settings, TLS certificates, and keys have been automatically -# generated to configure Elasticsearch security features on 26-08-2025 14:51:23 -# -# -------------------------------------------------------------------------------- - -# Enable security features -xpack.security.enabled: false - -xpack.security.enrollment.enabled: false - -# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents -xpack.security.http.ssl: - enabled: false - keystore.path: certs/http.p12 - -# Enable encryption and mutual authentication between cluster nodes -xpack.security.transport.ssl: - enabled: false - verification_mode: certificate - keystore.path: certs/transport.p12 - truststore.path: certs/transport.p12 -# Create a new cluster with the current node only -# Additional nodes can still join the cluster later -cluster.initial_master_nodes: ["frankfurt.tixeltec.de"] - -# Allow HTTP API connections from anywhere -# Connections are encrypted and require user authentication -http.host: 0.0.0.0 - -# Allow other nodes to join the cluster from anywhere -# Connections are encrypted and mutually authenticated -transport.host: 0.0.0.0 - -#----------------------- END SECURITY AUTO CONFIGURATION ------------------------- diff --git a/configs/config.yaml b/configs/example-config.yaml similarity index 60% rename from configs/config.yaml rename to configs/example-config.yaml index 6190a5f..706b863 100644 --- a/configs/config.yaml +++ b/configs/example-config.yaml @@ -1,35 +1,74 @@ + +export: + enabled: true + batch_size: 100 + export_interval: "30s" + retry_attempts: 5 + retry_backoff: "10s" + health_check_interval: "60s" + +localstorage: + enabled: true + db_path: "./watch.db" + rotation: + max_sizes_bytes: 100 * 1024 * 1024 + max_age_hours: 24 + max_files: 3 + check_interval_minuntes: 5 + archive_dir: "" + elasticsearch: - url: "http://localhost:9200" - index: "tixel" - username: "elastic" - password: "79QQ4JGTa3R_nkqA=MxW" + enabled: true + url: "http://10.0.0.99:9200" + index: "watch" + username: "your-configured-user" + password: "your-super-secret-password" + api_key: "your-api-key" timeout: 30 web_service: enabled: true - host: "localhost" - port: 8080 + host: "0.0.0.0" + port: 9090 system_metrics: enabled: true collect_cpu: true collect_memory: true collect_disk: true - collect_network: false + collect_network: true disk_paths: - "/" - "/var" - "/home" network_interfaces: - - "eth0" - - "wlan0" + - "ens6" + collect_network_connections: true + collect_load_average: true + collect_tcp_stats: true + collect_filehandles: true + collect_disk_io: true + collect_network_latency: true + collect_bandwidth_usage: true + transfer_ports: 60003 + latency_test_hosts: "www.google.de" poll_interval_seconds: 30 +patterns_file: "./configs/patterns.yaml" logging: level: "info" file_path: "/var/log/system-monitor.log" +drain3: + enabled: true + state_dir: "./drain3_states" + depth: 4 + sim_th: 0.4 + max_children: 100 + max_clusters: 1000 + save_interval: 60 + services: - name: "nginx" service: "nginx.service" @@ -37,18 +76,6 @@ services: since_time: "" priority: "info" - - name: "tixstream" - service: "tixstream.service" - enabled: true - since_time: "" - priority: "debug" - - - name: "transfer-job-manager" - service: "transfer-job-manager.service" - enabled: true - since_time: "" - priority: "debug" - tools: - name: "nginx-access" log_file: "/var/log/nginx/access.log" @@ -82,9 +109,3 @@ tools: tid: "thread_id" message: "error_message" - - name: "nginx-tjm" - log_file: "/var/log/nginx/access_tjm.log" - enabled: true - buffer_size: 100 - format: - name: "custom" diff --git a/configs/example-patterns.yaml b/configs/example-patterns.yaml new file mode 100644 index 0000000..56ce629 --- /dev/null +++ b/configs/example-patterns.yaml @@ -0,0 +1,36 @@ +patterns: + common: + extractors: + - name: "syslog_header" + regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P[^\s]+) (?P[^:]+):\s*(?P.*)$' + fields: + syslog_timestamp: "time:Jan 02 15:04:05" + hostname: "string" + process_info: "string" + message_rest: "string" + + - name: "iso8601_timestamp" + regex: '(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)' + fields: + timestamp: "time:2006-01-02T15:04:05.000000Z" + + nginx: + extractors: + - name: "access_log" + regex: '^(?P\S+)\s+\S+\s+(?P\S+)\s+\[(?P[^\]]+)\]\s+"(?P[^"]+)"\s+(?P\d+)\s+(?P\d+|-)' + fields: + client_ip: "string" + remote_user: "string" + timestamp_nginx: "string" + request: "string" + status_code: "int" + bytes_sent: "int" + + my-app: + extractors: + - name: "app_log" + regex: '^\[(?P\w+)\] id=(?P\d+) duration=(?P\d+)ms' + fields: + level: "string" + request_id: "int" + duration_ms: "int" diff --git a/configs/patterns.yml b/configs/patterns.yml deleted file mode 100644 index 069a38c..0000000 --- a/configs/patterns.yml +++ /dev/null @@ -1,164 +0,0 @@ -patterns: - # =========================================================================== - # Common / Shared Patterns - # =========================================================================== - common: - extractors: - - name: "syslog_header" - regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P[^\s]+) (?P[^:]+):\s*(?P.*)$' - fields: - syslog_timestamp: "time:Jan 02 15:04:05" - hostname: "string" - process_info: "string" - message_rest: "string" - - - name: "timestamp_rfc3339" - regex: '(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)' - fields: - timestamp: "time:2006-01-02T15:04:05.000000Z" - - # =========================================================================== - # TIXstream Service - # Deckt ab: tsServicePattern, tsTransferIDPattern, tsDetailPattern1-4 - # =========================================================================== - tixstream: - extractors: - - name: "service_log_base" - regex: '^(?P\S+)\s+(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?P.*)' - fields: - log_level: "string" - timestamp: "time:2006-01-02 15:04:05.000000" - message: "string" - - - name: "transfer_id_extraction" - regex: '^(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?P.*)' - fields: - transfer_id: "string" - message: "string" - - - name: "transfer_start_in" - regex: 'in: Transfer start (?P\d+/\d+) buffers=(?P\d+) files=(?P\d+) size=(?P[0-9.]+) MByte chunksize=(?P\d+) streams=(?P\d+) target-datarate=(?P[0-9.]+) MByte/s protocol=(?P\w+) dest=(?P\S+) sender-id=(?P\S+)' - fields: - thread_info: "string" # z.B. "1/4" - Typisierung hier schwierig, also String - buffers: "int" - file_count: "int" - size_mb: "float" - chunk_size: "int" - streams: "int" - target_rate: "float" - protocol: "string" - destination: "string" - sender_id: "string" - direction: "string" # Wir können statische Felder im Parser injecten oder hier als "implizit" betrachten - - - name: "transfer_start_remote_out" - regex: 'out: Start remote transfer to (?P[^\s]+) request executed, duration=(?P[0-9.]+) s' - fields: - target: "string" - duration: "float" - - - name: "transfer_start_out" - regex: 'out: Transfer start (?P\d+/\d+) buffers=(?P\d+) files=(?P\d+) size=(?P[0-9.]+) MByte chunksize=(?P\d+) streams=(?P\d+) target-datarate=(?P[0-9.]+) MByte/s protocol=(?P\w+) src=(?P\S+) receiver=(?P\S+)' - fields: - thread_info: "string" - buffers: "int" - file_count: "int" - size_mb: "float" - chunk_size: "int" - streams: "int" - target_rate: "float" - protocol: "string" - source: "string" - receiver: "string" - - - name: "transfer_start_generic" - regex: 'out: Start transfer (?P\d+/\d+), src=(?P[^ ]*) dest=(?P[^ ]*) item\[0\]=(?P[^ ]*) count=(?P\d+)' - fields: - thread_info: "string" - source: "string" - destination: "string" - item0: "string" - count: "int" - - # =========================================================================== - # Transfer Job Manager (TJM) - # Deckt ab: tjmServicePattern, tjmTransferNamePattern, tjmTransferIDPattern1/2 - # =========================================================================== - transfer-job-manager: - extractors: - - name: "service_log_base" - regex: '^(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?P\S+)\s+(?P\d+).*?\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+(?P.*?)\s+:\s+(?P.*)' - fields: - timestamp: "time:2006-01-02 15:04:05.000" - log_level: "string" - pid: "int" - correlation_id: "string" - username: "string" - thread_id: "string" - java_class: "string" - message: "string" - - - name: "transfer_name_info" - regex: '^(?P\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (?P.*)$' - fields: - transfer_name_raw: "string" - message: "string" - - - name: "transfer_id_mid" - regex: '(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P.*)' - fields: - transfer_id: "string" - message: "string" - - - name: "transfer_id_prefixed" - regex: '(?P.*)(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P.*)' - fields: - prefix: "string" - transfer_id: "string" - message: "string" - - # =========================================================================== - # Access Manager & TCC - # Deckt ab: amServicePattern, tccServicePattern - # =========================================================================== - access-manager: - extractors: - - name: "spring_boot_log" - regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P\w+)\s+(?P\d+)\s+---\s+\[\s*(?P[^\]]*)\]\s+(?P[\w\.]+)\s*:\s+(?P.*)$' - fields: - timestamp: "time:2006-01-02T15:04:05.000000Z" - log_level: "string" - pid: "int" - thread_id: "string" - logger: "string" - message: "string" - - tixel-control-center: - extractors: - - name: "spring_boot_log" - regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P\w+)\s+(?P\d+)\s+---\s+\[\s*(?P[^\]]*)\]\s+(?P[\w\.]+)\s*:\s+(?P.*)$' - fields: - timestamp: "time:2006-01-02T15:04:05.000000Z" - log_level: "string" - pid: "int" - thread_id: "string" - logger: "string" - message: "string" - - # =========================================================================== - # Nginx - # Deckt ab: nginxAccessPattern - # =========================================================================== - nginx: - extractors: - - name: "access_log" - regex: '^(?P\S+)\s+\S+\s+(?P\S+)\s+\[(?P[^\]]+)\]\s+"(?P[^"]+)"\s+(?P\d+)\s+(?P\d+|-)\s*(?:"(?P[^"]*)"\s+"(?P[^"]*)")?' - fields: - client_ip: "string" - remote_user: "string" - timestamp_nginx: "string" - request: "string" - status_code: "int" - bytes_sent: "int" - referer: "string" - user_agent: "string" diff --git a/elasticsearch.go b/elasticsearch.go index e40677a..b897cb8 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -70,7 +70,7 @@ func (esc *ElasticsearchClient) SendBatch(baseIndex string, entries []models.Log var body strings.Builder for _, entry := range entries { - indexName := "tixel" + indexName := "watch" indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName) body.WriteString(indexLine) @@ -117,7 +117,7 @@ func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics mode return fmt.Errorf("JSON marshalling error: %w", err) } - systemIndex := "tixel" + systemIndex := "watch" ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/install.sh b/install.sh deleted file mode 100755 index c203051..0000000 --- a/install.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -set -e - -ES_VERSION="9.1.2" -ES_DEB_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-amd64.deb" -ES_RPM_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-x86_64.rpm" -ES_CONFIG_DIR="/etc/elasticsearch" -ES_JVM_OPTIONS="/etc/elasticsearch/jvm.options" -ES_JVM_OPTIONS_D="/etc/elasticsearch/jvm.options.d" -GO_SERVICE_NAME="tixel-watch" -GO_INSTALL_TARGET="/opt/tixel/tixel-watch" - -install_es_deb() { - echo "Installing Elasticsearch (Debian package)..." - wget "${ES_DEB_URL}" -O elasticsearch.deb - sudo dpkg -i elasticsearch.deb - sudo apt-get install -f -y -} - -install_es_rpm() { - echo "Installing Elasticsearch (RPM package)..." - wget "${ES_RPM_URL}" -O elasticsearch.rpm - sudo rpm --install elasticsearch.rpm -} - -setup_configuration() { - echo "Copying Elasticsearch configuration files..." - sudo cp ./configs/elasticsearch.yml "${ES_CONFIG_DIR}/elasticsearch.yml" - sudo cp ./configs/jvm.options "${ES_JVM_OPTIONS}" - sudo cp -r ./configs/jvm.options.d "${ES_JVM_OPTIONS_D}" - sudo chown root:elasticsearch "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}" - sudo chmod 640 "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}" -} - -setup_tixel_watch_service() { - echo "Setting up tixel-watch systemd service..." - if [ ! -d ${GO_INSTALL_TARGET} ]; then - mkdir -p ${GO_INSTALL_TARGET} - fi - sudo cp ./tixel-watch "$GO_INSTALL_TARGET"/ - sudo cp ./configs/config.yaml "$GO_INSTALL_TARGET"/ - sudo cp ./${GO_SERVICE_NAME}.service /etc/systemd/system/${GO_SERVICE_NAME}.service - sudo systemctl daemon-reload - sudo systemctl enable "${GO_SERVICE_NAME}" -} - -start_services() { - echo "Enabling and starting Elasticsearch service..." - sudo systemctl enable elasticsearch - sudo systemctl start elasticsearch - echo "Starting tixel-watch service..." - sudo systemctl start "${GO_SERVICE_NAME}" -} - -main() { - if command -v apt-get &>/dev/null; then - install_es_deb - elif command -v yum &>/dev/null || command -v dnf &>/dev/null; then - install_es_rpm - else - echo "Unsupported package manager. Aborting." - exit 1 - fi - - setup_configuration - setup_tixel_watch_service - start_services - echo "All done." -} - -main "$@" diff --git a/tixel-watch.service b/tixel-watch.service deleted file mode 100644 index d954224..0000000 --- a/tixel-watch.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=tixel-watch -After=network.target - -[Service] -Type=simple -User=tixstream -Group=tixstream -WorkingDirectory=/opt/tixel/tixel-watch -ExecStart=/opt/tixel/tixel-watch/tixel-watch -Restart=on-failure -RestartSec=5 -StandardOutput=syslog -StandardError=syslog -SyslogIdentifier=tixel-watch - -[Install] -WantedBy=multi-user.target - diff --git a/web_service.go b/web_service.go index 305a30d..d276ba5 100644 --- a/web_service.go +++ b/web_service.go @@ -98,7 +98,7 @@ func (ws *WebService) handleHealth(w http.ResponseWriter, r *http.Request) { status["storage_status"] = "healthy" } statusMap := make(map[string]any) - statusMap["tixel-watch"] = status + statusMap["watch"] = status for _, service := range ws.config.Services { statusCommand := []string{"sudo", "systemctl", "status", service.Name, "--no-pager"} @@ -160,7 +160,7 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) { return } - filename := fmt.Sprintf("tixel_export_%s.json", time.Now().Format("20060102_150405")) + filename := fmt.Sprintf("watch_export_%s.json", time.Now().Format("20060102_150405")) w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename)) @@ -169,7 +169,7 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) { "timestamp": time.Now(), "entry_count": len(entries), "query": query, - "exported_by": "tixel-watch", + "exported_by": "watch", }, "entries": entries, }