git/watch-tool
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: clean up and add example config and pattern

Browse source
This commit is contained in:
Patryk Hegenberg 2026-01-18 17:51:23 +01:00
parent 07798189a2
commit 17723de72f
10 changed files with 96 additions and 430 deletions
Download patch file Download diff file Expand all files Collapse all files

23
build.sh
View file

@ -1,23 +0,0 @@
#!/bin/bash
set -e
PACKAGE_DIR="./packages"
PACKAGE_NAME="./tixel-watch"
if [ -d "${PACKAGE_DIR}" ]; then
rm -rf "${PACKAGE_DIR:?}/"*
else
mkdir -p "${PACKAGE_DIR}"
fi
if [ -d "${PACKAGE_NAME}" ]; then
rm -rf "${PACKAGE_NAME:?}"
fi
CGO_ENABLED=0 go build -ldflags="-s -w" -o "$PACKAGE_DIR"/tixel-watch -buildvcs .
cp -r ./tixel-watch.service $PACKAGE_DIR/
cp -r ./configs/ $PACKAGE_DIR/
cp -r ./install.sh $PACKAGE_DIR/
mv $PACKAGE_DIR tixel-watch
tar -czvf tixel-watch.tar.gz ./tixel-watch

9
config.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"log/slog"
"os"
"regexp"
"time"
@ -156,7 +157,7 @@ func setConfigDefaults() {
viper.SetDefault("export.retry_backoff", "5s")
viper.SetDefault("export.health_check_interval", "60s")
viper.SetDefault("localstorage.enabled", true)
viper.SetDefault("localstorage.db_path", "./tixel_watch.db")
viper.SetDefault("localstorage.db_path", "./watch.db")
viper.SetDefault("localstorage.rotation.max_size_bytes", int64(100*1024*1024))
viper.SetDefault("localstorage.rotation.max_age_hours", 24)
viper.SetDefault("localstorage.rotation.max_files", 7)
@ -171,9 +172,13 @@ func setConfigDefaults() {
}
func LoadConfig() (*Config, error) {
home, err := os.UserConfigDir()
if err != nil {
return nil, fmt.Errorf("unable to get user config dir: %w", err)
}
viper.SetConfigName("config")
viper.AddConfigPath(".")
viper.AddConfigPath("/opt/tixel/tixel-watch/")
viper.AddConfigPath(home)
viper.SetConfigType("yaml")
setConfigDefaults()

119
configs/elasticsearch.yml
View file

@ -1,119 +0,0 @@
# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
# Before you set out to tweak and tune the configuration, make sure you
# understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: tixel-elastic
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
#node.name: node-1
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /var/lib/elasticsearch
#
# Path to log files:
#
path.logs: /var/log/elasticsearch
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# By default Elasticsearch is only accessible on localhost. Set a different
# address here to expose this node on the network:
#
network.host: 0.0.0.0
#
# By default Elasticsearch listens for HTTP traffic on the first free port it
# finds starting at 9200. Set a specific HTTP port here:
#
#http.port: 9200
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when this node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
#discovery.seed_hosts: ["host1", "host2"]
#
# Bootstrap the cluster using an initial set of master-eligible nodes:
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
#
# For more information, consult the discovery and cluster formation module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Allow wildcard deletion of indices:
#
#action.destructive_requires_name: false
#----------------------- BEGIN SECURITY AUTO CONFIGURATION -----------------------
#
# The following settings, TLS certificates, and keys have been automatically
# generated to configure Elasticsearch security features on 26-08-2025 14:51:23
#
# --------------------------------------------------------------------------------
# Enable security features
xpack.security.enabled: false
xpack.security.enrollment.enabled: false
# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents
xpack.security.http.ssl:
enabled: false
keystore.path: certs/http.p12
# Enable encryption and mutual authentication between cluster nodes
xpack.security.transport.ssl:
enabled: false
verification_mode: certificate
keystore.path: certs/transport.p12
truststore.path: certs/transport.p12
# Create a new cluster with the current node only
# Additional nodes can still join the cluster later
cluster.initial_master_nodes: ["frankfurt.tixeltec.de"]
# Allow HTTP API connections from anywhere
# Connections are encrypted and require user authentication
http.host: 0.0.0.0
# Allow other nodes to join the cluster from anywhere
# Connections are encrypted and mutually authenticated
transport.host: 0.0.0.0
#----------------------- END SECURITY AUTO CONFIGURATION -------------------------

75
configs/config.yaml → configs/example-config.yaml
View file

@ -1,35 +1,74 @@
export:
enabled: true
batch_size: 100
export_interval: "30s"
retry_attempts: 5
retry_backoff: "10s"
health_check_interval: "60s"
localstorage:
enabled: true
db_path: "./watch.db"
rotation:
max_sizes_bytes: 100 * 1024 * 1024
max_age_hours: 24
max_files: 3
check_interval_minuntes: 5
archive_dir: ""
elasticsearch:
url: "http://localhost:9200"
index: "tixel"
username: "elastic"
password: "79QQ4JGTa3R_nkqA=MxW"
enabled: true
url: "http://10.0.0.99:9200"
index: "watch"
username: "your-configured-user"
password: "your-super-secret-password"
api_key: "your-api-key"
timeout: 30
web_service:
enabled: true
host: "localhost"
port: 8080
host: "0.0.0.0"
port: 9090
system_metrics:
enabled: true
collect_cpu: true
collect_memory: true
collect_disk: true
collect_network: false
collect_network: true
disk_paths:
- "/"
- "/var"
- "/home"
network_interfaces:
- "eth0"
- "wlan0"
- "ens6"
collect_network_connections: true
collect_load_average: true
collect_tcp_stats: true
collect_filehandles: true
collect_disk_io: true
collect_network_latency: true
collect_bandwidth_usage: true
transfer_ports: 60003
latency_test_hosts: "www.google.de"
poll_interval_seconds: 30
patterns_file: "./configs/patterns.yaml"
logging:
level: "info"
file_path: "/var/log/system-monitor.log"
drain3:
enabled: true
state_dir: "./drain3_states"
depth: 4
sim_th: 0.4
max_children: 100
max_clusters: 1000
save_interval: 60
services:
- name: "nginx"
service: "nginx.service"
@ -37,18 +76,6 @@ services:
since_time: ""
priority: "info"
- name: "tixstream"
service: "tixstream.service"
enabled: true
since_time: ""
priority: "debug"
- name: "transfer-job-manager"
service: "transfer-job-manager.service"
enabled: true
since_time: ""
priority: "debug"
tools:
- name: "nginx-access"
log_file: "/var/log/nginx/access.log"
@ -82,9 +109,3 @@ tools:
tid: "thread_id"
message: "error_message"
- name: "nginx-tjm"
log_file: "/var/log/nginx/access_tjm.log"
enabled: true
buffer_size: 100
format:
name: "custom"

36
configs/example-patterns.yaml Normal file
View file

@ -0,0 +1,36 @@
patterns:
common:
extractors:
- name: "syslog_header"
regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+) (?P<process_info>[^:]+):\s*(?P<message_rest>.*)$'
fields:
syslog_timestamp: "time:Jan 02 15:04:05"
hostname: "string"
process_info: "string"
message_rest: "string"
- name: "iso8601_timestamp"
regex: '(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)'
fields:
timestamp: "time:2006-01-02T15:04:05.000000Z"
nginx:
extractors:
- name: "access_log"
regex: '^(?P<client_ip>\S+)\s+\S+\s+(?P<remote_user>\S+)\s+\[(?P<timestamp_nginx>[^\]]+)\]\s+"(?P<request>[^"]+)"\s+(?P<status_code>\d+)\s+(?P<bytes_sent>\d+|-)'
fields:
client_ip: "string"
remote_user: "string"
timestamp_nginx: "string"
request: "string"
status_code: "int"
bytes_sent: "int"
my-app:
extractors:
- name: "app_log"
regex: '^\[(?P<level>\w+)\] id=(?P<request_id>\d+) duration=(?P<duration_ms>\d+)ms'
fields:
level: "string"
request_id: "int"
duration_ms: "int"

164
configs/patterns.yml
View file

@ -1,164 +0,0 @@
patterns:
# ===========================================================================
# Common / Shared Patterns
# ===========================================================================
common:
extractors:
- name: "syslog_header"
regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+) (?P<process_info>[^:]+):\s*(?P<message_rest>.*)$'
fields:
syslog_timestamp: "time:Jan 02 15:04:05"
hostname: "string"
process_info: "string"
message_rest: "string"
- name: "timestamp_rfc3339"
regex: '(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)'
fields:
timestamp: "time:2006-01-02T15:04:05.000000Z"
# ===========================================================================
# TIXstream Service
# Deckt ab: tsServicePattern, tsTransferIDPattern, tsDetailPattern1-4
# ===========================================================================
tixstream:
extractors:
- name: "service_log_base"
regex: '^(?P<log_level>\S+)\s+(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?P<message>.*)'
fields:
log_level: "string"
timestamp: "time:2006-01-02 15:04:05.000000"
message: "string"
- name: "transfer_id_extraction"
regex: '^(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?P<message>.*)'
fields:
transfer_id: "string"
message: "string"
- name: "transfer_start_in"
regex: 'in: Transfer start (?P<thread_info>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<file_count>\d+) size=(?P<size_mb>[0-9.]+) MByte chunksize=(?P<chunk_size>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_rate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<destination>\S+) sender-id=(?P<sender_id>\S+)'
fields:
thread_info: "string" # z.B. "1/4" - Typisierung hier schwierig, also String
buffers: "int"
file_count: "int"
size_mb: "float"
chunk_size: "int"
streams: "int"
target_rate: "float"
protocol: "string"
destination: "string"
sender_id: "string"
direction: "string" # Wir können statische Felder im Parser injecten oder hier als "implizit" betrachten
- name: "transfer_start_remote_out"
regex: 'out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s'
fields:
target: "string"
duration: "float"
- name: "transfer_start_out"
regex: 'out: Transfer start (?P<thread_info>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<file_count>\d+) size=(?P<size_mb>[0-9.]+) MByte chunksize=(?P<chunk_size>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_rate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<source>\S+) receiver=(?P<receiver>\S+)'
fields:
thread_info: "string"
buffers: "int"
file_count: "int"
size_mb: "float"
chunk_size: "int"
streams: "int"
target_rate: "float"
protocol: "string"
source: "string"
receiver: "string"
- name: "transfer_start_generic"
regex: 'out: Start transfer (?P<thread_info>\d+/\d+), src=(?P<source>[^ ]*) dest=(?P<destination>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)'
fields:
thread_info: "string"
source: "string"
destination: "string"
item0: "string"
count: "int"
# ===========================================================================
# Transfer Job Manager (TJM)
# Deckt ab: tjmServicePattern, tjmTransferNamePattern, tjmTransferIDPattern1/2
# ===========================================================================
transfer-job-manager:
extractors:
- name: "service_log_base"
regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?P<log_level>\S+)\s+(?P<pid>\d+).*?\[(?P<correlation_id>[^\]]*)\]\s+\[(?P<username>[^\]]*)\]\s+\[(?P<thread_id>[^\]]*)\]\s+(?P<java_class>.*?)\s+:\s+(?P<message>.*)'
fields:
timestamp: "time:2006-01-02 15:04:05.000"
log_level: "string"
pid: "int"
correlation_id: "string"
username: "string"
thread_id: "string"
java_class: "string"
message: "string"
- name: "transfer_name_info"
regex: '^(?P<transfer_name_raw>\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (?P<message>.*)$'
fields:
transfer_name_raw: "string"
message: "string"
- name: "transfer_id_mid"
regex: '(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)'
fields:
transfer_id: "string"
message: "string"
- name: "transfer_id_prefixed"
regex: '(?P<prefix>.*)(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)'
fields:
prefix: "string"
transfer_id: "string"
message: "string"
# ===========================================================================
# Access Manager & TCC
# Deckt ab: amServicePattern, tccServicePattern
# ===========================================================================
access-manager:
extractors:
- name: "spring_boot_log"
regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P<log_level>\w+)\s+(?P<pid>\d+)\s+---\s+\[\s*(?P<thread_id>[^\]]*)\]\s+(?P<logger>[\w\.]+)\s*:\s+(?P<message>.*)$'
fields:
timestamp: "time:2006-01-02T15:04:05.000000Z"
log_level: "string"
pid: "int"
thread_id: "string"
logger: "string"
message: "string"
tixel-control-center:
extractors:
- name: "spring_boot_log"
regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P<log_level>\w+)\s+(?P<pid>\d+)\s+---\s+\[\s*(?P<thread_id>[^\]]*)\]\s+(?P<logger>[\w\.]+)\s*:\s+(?P<message>.*)$'
fields:
timestamp: "time:2006-01-02T15:04:05.000000Z"
log_level: "string"
pid: "int"
thread_id: "string"
logger: "string"
message: "string"
# ===========================================================================
# Nginx
# Deckt ab: nginxAccessPattern
# ===========================================================================
nginx:
extractors:
- name: "access_log"
regex: '^(?P<client_ip>\S+)\s+\S+\s+(?P<remote_user>\S+)\s+\[(?P<timestamp_nginx>[^\]]+)\]\s+"(?P<request>[^"]+)"\s+(?P<status_code>\d+)\s+(?P<bytes_sent>\d+|-)\s*(?:"(?P<referer>[^"]*)"\s+"(?P<user_agent>[^"]*)")?'
fields:
client_ip: "string"
remote_user: "string"
timestamp_nginx: "string"
request: "string"
status_code: "int"
bytes_sent: "int"
referer: "string"
user_agent: "string"

4
elasticsearch.go
View file

@ -70,7 +70,7 @@ func (esc *ElasticsearchClient) SendBatch(baseIndex string, entries []models.Log
var body strings.Builder
for _, entry := range entries {
indexName := "tixel"
indexName := "watch"
indexLine := fmt.Sprintf(`{"index":{"_index":"%s"}}`, indexName)
body.WriteString(indexLine)
@ -117,7 +117,7 @@ func (esc *ElasticsearchClient) SendSystemMetrics(baseIndex string, metrics mode
return fmt.Errorf("JSON marshalling error: %w", err)
}
systemIndex := "tixel"
systemIndex := "watch"
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()

71
install.sh
View file

@ -1,71 +0,0 @@
#!/bin/bash
set -e
ES_VERSION="9.1.2"
ES_DEB_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-amd64.deb"
ES_RPM_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-x86_64.rpm"
ES_CONFIG_DIR="/etc/elasticsearch"
ES_JVM_OPTIONS="/etc/elasticsearch/jvm.options"
ES_JVM_OPTIONS_D="/etc/elasticsearch/jvm.options.d"
GO_SERVICE_NAME="tixel-watch"
GO_INSTALL_TARGET="/opt/tixel/tixel-watch"
install_es_deb() {
echo "Installing Elasticsearch (Debian package)..."
wget "${ES_DEB_URL}" -O elasticsearch.deb
sudo dpkg -i elasticsearch.deb
sudo apt-get install -f -y
}
install_es_rpm() {
echo "Installing Elasticsearch (RPM package)..."
wget "${ES_RPM_URL}" -O elasticsearch.rpm
sudo rpm --install elasticsearch.rpm
}
setup_configuration() {
echo "Copying Elasticsearch configuration files..."
sudo cp ./configs/elasticsearch.yml "${ES_CONFIG_DIR}/elasticsearch.yml"
sudo cp ./configs/jvm.options "${ES_JVM_OPTIONS}"
sudo cp -r ./configs/jvm.options.d "${ES_JVM_OPTIONS_D}"
sudo chown root:elasticsearch "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}"
sudo chmod 640 "${ES_CONFIG_DIR}/elasticsearch.yml" "${ES_JVM_OPTIONS}"
}
setup_tixel_watch_service() {
echo "Setting up tixel-watch systemd service..."
if [ ! -d ${GO_INSTALL_TARGET} ]; then
mkdir -p ${GO_INSTALL_TARGET}
fi
sudo cp ./tixel-watch "$GO_INSTALL_TARGET"/
sudo cp ./configs/config.yaml "$GO_INSTALL_TARGET"/
sudo cp ./${GO_SERVICE_NAME}.service /etc/systemd/system/${GO_SERVICE_NAME}.service
sudo systemctl daemon-reload
sudo systemctl enable "${GO_SERVICE_NAME}"
}
start_services() {
echo "Enabling and starting Elasticsearch service..."
sudo systemctl enable elasticsearch
sudo systemctl start elasticsearch
echo "Starting tixel-watch service..."
sudo systemctl start "${GO_SERVICE_NAME}"
}
main() {
if command -v apt-get &>/dev/null; then
install_es_deb
elif command -v yum &>/dev/null || command -v dnf &>/dev/null; then
install_es_rpm
else
echo "Unsupported package manager. Aborting."
exit 1
fi
setup_configuration
setup_tixel_watch_service
start_services
echo "All done."
}
main "$@"

19
tixel-watch.service
View file

@ -1,19 +0,0 @@
[Unit]
Description=tixel-watch
After=network.target
[Service]
Type=simple
User=tixstream
Group=tixstream
WorkingDirectory=/opt/tixel/tixel-watch
ExecStart=/opt/tixel/tixel-watch/tixel-watch
Restart=on-failure
RestartSec=5
StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=tixel-watch
[Install]
WantedBy=multi-user.target

6
web_service.go
View file

@ -98,7 +98,7 @@ func (ws *WebService) handleHealth(w http.ResponseWriter, r *http.Request) {
status["storage_status"] = "healthy"
}
statusMap := make(map[string]any)
statusMap["tixel-watch"] = status
statusMap["watch"] = status
for _, service := range ws.config.Services {
statusCommand := []string{"sudo", "systemctl", "status", service.Name, "--no-pager"}
@ -160,7 +160,7 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
return
}
filename := fmt.Sprintf("tixel_export_%s.json", time.Now().Format("20060102_150405"))
filename := fmt.Sprintf("watch_export_%s.json", time.Now().Format("20060102_150405"))
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
@ -169,7 +169,7 @@ func (ws *WebService) handleExport(w http.ResponseWriter, r *http.Request) {
"timestamp": time.Now(),
"entry_count": len(entries),
"query": query,
"exported_by": "tixel-watch",
"exported_by": "watch",
},
"entries": entries,
}