patterns: # =========================================================================== # Common / Shared Patterns # =========================================================================== common: extractors: - name: "syslog_header" regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P[^\s]+) (?P[^:]+):\s*(?P.*)$' fields: syslog_timestamp: "time:Jan 02 15:04:05" hostname: "string" process_info: "string" message_rest: "string" - name: "timestamp_rfc3339" regex: '(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)' fields: timestamp: "time:2006-01-02T15:04:05.000000Z" # =========================================================================== # TIXstream Service # Deckt ab: tsServicePattern, tsTransferIDPattern, tsDetailPattern1-4 # =========================================================================== tixstream: extractors: - name: "service_log_base" regex: '^(?P\S+)\s+(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?P.*)' fields: log_level: "string" timestamp: "time:2006-01-02 15:04:05.000000" message: "string" - name: "transfer_id_extraction" regex: '^(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?P.*)' fields: transfer_id: "string" message: "string" - name: "transfer_start_in" regex: 'in: Transfer start (?P\d+/\d+) buffers=(?P\d+) files=(?P\d+) size=(?P[0-9.]+) MByte chunksize=(?P\d+) streams=(?P\d+) target-datarate=(?P[0-9.]+) MByte/s protocol=(?P\w+) dest=(?P\S+) sender-id=(?P\S+)' fields: thread_info: "string" # z.B. "1/4" - Typisierung hier schwierig, also String buffers: "int" file_count: "int" size_mb: "float" chunk_size: "int" streams: "int" target_rate: "float" protocol: "string" destination: "string" sender_id: "string" direction: "string" # Wir können statische Felder im Parser injecten oder hier als "implizit" betrachten - name: "transfer_start_remote_out" regex: 'out: Start remote transfer to (?P[^\s]+) request executed, duration=(?P[0-9.]+) s' fields: target: "string" duration: "float" - name: "transfer_start_out" regex: 'out: Transfer start (?P\d+/\d+) buffers=(?P\d+) files=(?P\d+) size=(?P[0-9.]+) MByte chunksize=(?P\d+) streams=(?P\d+) target-datarate=(?P[0-9.]+) MByte/s protocol=(?P\w+) src=(?P\S+) receiver=(?P\S+)' fields: thread_info: "string" buffers: "int" file_count: "int" size_mb: "float" chunk_size: "int" streams: "int" target_rate: "float" protocol: "string" source: "string" receiver: "string" - name: "transfer_start_generic" regex: 'out: Start transfer (?P\d+/\d+), src=(?P[^ ]*) dest=(?P[^ ]*) item\[0\]=(?P[^ ]*) count=(?P\d+)' fields: thread_info: "string" source: "string" destination: "string" item0: "string" count: "int" # =========================================================================== # Transfer Job Manager (TJM) # Deckt ab: tjmServicePattern, tjmTransferNamePattern, tjmTransferIDPattern1/2 # =========================================================================== transfer-job-manager: extractors: - name: "service_log_base" regex: '^(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?P\S+)\s+(?P\d+).*?\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+(?P.*?)\s+:\s+(?P.*)' fields: timestamp: "time:2006-01-02 15:04:05.000" log_level: "string" pid: "int" correlation_id: "string" username: "string" thread_id: "string" java_class: "string" message: "string" - name: "transfer_name_info" regex: '^(?P\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (?P.*)$' fields: transfer_name_raw: "string" message: "string" - name: "transfer_id_mid" regex: '(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P.*)' fields: transfer_id: "string" message: "string" - name: "transfer_id_prefixed" regex: '(?P.*)(?P\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P.*)' fields: prefix: "string" transfer_id: "string" message: "string" # =========================================================================== # Access Manager & TCC # Deckt ab: amServicePattern, tccServicePattern # =========================================================================== access-manager: extractors: - name: "spring_boot_log" regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P\w+)\s+(?P\d+)\s+---\s+\[\s*(?P[^\]]*)\]\s+(?P[\w\.]+)\s*:\s+(?P.*)$' fields: timestamp: "time:2006-01-02T15:04:05.000000Z" log_level: "string" pid: "int" thread_id: "string" logger: "string" message: "string" tixel-control-center: extractors: - name: "spring_boot_log" regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P\w+)\s+(?P\d+)\s+---\s+\[\s*(?P[^\]]*)\]\s+(?P[\w\.]+)\s*:\s+(?P.*)$' fields: timestamp: "time:2006-01-02T15:04:05.000000Z" log_level: "string" pid: "int" thread_id: "string" logger: "string" message: "string" # =========================================================================== # Nginx # Deckt ab: nginxAccessPattern # =========================================================================== nginx: extractors: - name: "access_log" regex: '^(?P\S+)\s+\S+\s+(?P\S+)\s+\[(?P[^\]]+)\]\s+"(?P[^"]+)"\s+(?P\d+)\s+(?P\d+|-)\s*(?:"(?P[^"]*)"\s+"(?P[^"]*)")?' fields: client_ip: "string" remote_user: "string" timestamp_nginx: "string" request: "string" status_code: "int" bytes_sent: "int" referer: "string" user_agent: "string"