add Final Infrastructure Setup
This commit is contained in:
commit
7733dde658
174 changed files with 204949 additions and 0 deletions
29
infrastructure/ansible/roles/blackhole/tasks/main.yml
Normal file
29
infrastructure/ansible/roles/blackhole/tasks/main.yml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
- name: Kopiere Blackhole Script
|
||||
template:
|
||||
src: blackhole.sh.j2
|
||||
dest: /usr/local/bin/blackhole.sh
|
||||
mode: "0755"
|
||||
|
||||
- name: Deploye Systemd Service für Blackhole
|
||||
copy:
|
||||
dest: /etc/systemd/system/blackhole.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Thesis Blackhole Service (Cleanup)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/blackhole.sh
|
||||
Restart=always
|
||||
User=root
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
- name: Starte Blackhole Service
|
||||
systemd:
|
||||
name: blackhole
|
||||
state: started
|
||||
enabled: yes
|
||||
daemon_reload: yes
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
#!/bin/bash
|
||||
|
||||
TARGET_DIR="/local_testdata/test-destination"
|
||||
|
||||
THRESHOLD=60
|
||||
TARGET_USAGE=50
|
||||
|
||||
SLEEP_INTERVAL=2
|
||||
DELETE_BATCH=200
|
||||
SAFE_MINUTES=5
|
||||
|
||||
echo "[$(date)] Blackhole Monitor gestartet für $TARGET_DIR"
|
||||
|
||||
while true; do
|
||||
|
||||
CURRENT_USAGE=$(df -P "$TARGET_DIR" | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
|
||||
if (( CURRENT_USAGE > THRESHOLD )); then
|
||||
echo "[$(date)] Disk usage ${CURRENT_USAGE}% -> starte Cleanup"
|
||||
|
||||
while (( CURRENT_USAGE > TARGET_USAGE )); do
|
||||
|
||||
find "$TARGET_DIR" \
|
||||
-type f \
|
||||
-mmin +$SAFE_MINUTES \
|
||||
-print0 \
|
||||
| head -z -n $DELETE_BATCH \
|
||||
| xargs -0 -r rm -f
|
||||
|
||||
CURRENT_USAGE=$(df -P "$TARGET_DIR" | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
|
||||
echo "[$(date)] aktuelle Auslastung: ${CURRENT_USAGE}%"
|
||||
|
||||
if ! find "$TARGET_DIR" -type f -mmin +$SAFE_MINUTES | head -n 1 | grep -q .; then
|
||||
echo "[$(date)] Keine alten Dateien mehr zum Löschen"
|
||||
break
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
echo "[$(date)] Cleanup beendet"
|
||||
fi
|
||||
|
||||
sleep "$SLEEP_INTERVAL"
|
||||
done
|
||||
60
infrastructure/ansible/roles/common/tasks/main.yml
Normal file
60
infrastructure/ansible/roles/common/tasks/main.yml
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
---
|
||||
- name: Update apt cache
|
||||
apt:
|
||||
update_cache: yes
|
||||
cache_valid_time: 3600
|
||||
|
||||
- name: Install base-packets for all nodes
|
||||
apt:
|
||||
name:
|
||||
- ca-certificates
|
||||
- curl
|
||||
- wget
|
||||
- vim
|
||||
- tree
|
||||
- git
|
||||
- unzip
|
||||
- htop
|
||||
- sysstat
|
||||
- net-tools
|
||||
- python3-pip
|
||||
- python3-venv
|
||||
- curl
|
||||
- jq
|
||||
- bc
|
||||
- sysstat
|
||||
- stress-ng
|
||||
state: present
|
||||
|
||||
- name: Install python-libraries (System-Level)
|
||||
apt:
|
||||
name:
|
||||
- python3-pymysql
|
||||
state: present
|
||||
|
||||
- name: Ensure NTP/Timesync is running
|
||||
service:
|
||||
name: systemd-timesyncd
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
- name: Stelle sicher, dass das Universe Repository aktiv ist (für OpenJDK 8)
|
||||
apt_repository:
|
||||
repo: "deb http://archive.ubuntu.com/ubuntu/ noble universe"
|
||||
state: present
|
||||
|
||||
- name: Installiere Java 8 (OpenJDK JRE Headless)
|
||||
apt:
|
||||
name: openjdk-8-jre-headless
|
||||
state: present
|
||||
update_cache: yes
|
||||
|
||||
- name: Add hosts to /etc/hosts file on Linux
|
||||
lineinfile:
|
||||
dest: /etc/hosts
|
||||
regexp: "^.*{{ item.value.ip }}.*{{ item.value.hostname }}.*$"
|
||||
line: "{{ item.value.ip }} {{ item.value.hostname }}"
|
||||
state: present
|
||||
when: ansible_os_family != "Windows"
|
||||
become: true
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
127.0.0.1 localhost
|
||||
|
||||
# Automatisch generiert aus host_config
|
||||
{% for hostname, config in configs.host_config.items() %}
|
||||
{{ config.ip }} {{ config.hostname }} {{ hostname }}
|
||||
{% endfor %}
|
||||
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
- include_tasks: sftp-ubuntu20.yml
|
||||
when: ansible_distribution == "Ubuntu"
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
---
|
||||
- name: add sftp user
|
||||
user:
|
||||
name: "{{ sftpuser }}"
|
||||
group: "{{ sftpgroup }}"
|
||||
password: "{{ ftphash }}"
|
||||
become: true
|
||||
|
||||
- name: Make sure password authentification is enabled
|
||||
lineinfile:
|
||||
path: /etc/ssh/sshd_config
|
||||
regexp: "^PasswordAuthentication"
|
||||
line: "PasswordAuthentication yes"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Add sftp config to sshd_config
|
||||
blockinfile:
|
||||
path: /etc/ssh/sshd_config
|
||||
block: |
|
||||
Match User {{ sftpuser }}
|
||||
ForceCommand internal-sftp
|
||||
PasswordAuthentication yes
|
||||
PermitTunnel no
|
||||
AllowAgentForwarding no
|
||||
AllowTcpForwarding no
|
||||
X11Forwarding no
|
||||
become: true
|
||||
|
||||
- name: restart sshd-server
|
||||
service:
|
||||
name: sshd
|
||||
state: restarted
|
||||
become: true
|
||||
when: ansible_distribution == "Ubuntu" and (ansible_facts['distribution_version'] == "22.04" or ansible_facts['distribution_version'] == "20.04")
|
||||
|
||||
- name: restart ssh-server
|
||||
service:
|
||||
name: ssh
|
||||
state: restarted
|
||||
become: true
|
||||
when: ansible_distribution == "Ubuntu" and ansible_facts['distribution_version'] == "24.04"
|
||||
33
infrastructure/ansible/roles/mft-codemeter/tasks/main.yml
Normal file
33
infrastructure/ansible/roles/mft-codemeter/tasks/main.yml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
|
||||
- name: Codemeter is started
|
||||
service:
|
||||
name: codemeter
|
||||
state: restarted
|
||||
|
||||
- name: Start tixstream to find out if a new license has to be installed
|
||||
command: "/opt/tixel/tixstream/bin/tixstream --check-license --license-type 380"
|
||||
register: license_ok
|
||||
ignore_errors: True
|
||||
|
||||
- name: Find CodeMeter license
|
||||
local_action: find paths="{{ configs.deployment_dir }}" patterns="*.WibuCmRaU" file_type=file
|
||||
become: false
|
||||
register: all_codemeter_licenses
|
||||
when: license_ok is failed
|
||||
|
||||
- name: Get latest CodeMeter license
|
||||
set_fact:
|
||||
codemeter_license: "{{ all_codemeter_licenses.files | sort(attribute='mtime',reverse=true) | first }}"
|
||||
when: license_ok is failed
|
||||
|
||||
- name: copy license file
|
||||
copy: src={{ codemeter_license.path }} dest={{ remote_deployment_dir }}
|
||||
when: license_ok is failed
|
||||
|
||||
- name: Install license
|
||||
shell: cmu -i -f {{ remote_deployment_dir }}/{{ codemeter_license.path | basename }}
|
||||
when: license_ok is failed
|
||||
ignore_errors: True
|
||||
|
||||
# vim:ft=ansible
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
---
|
||||
# NOTE: There are also ansible modules that help generating certificates
|
||||
# http://docs.ansible.com/ansible/latest/list_of_crypto_modules.html
|
||||
#
|
||||
# but I trust direct calls to openssl more..
|
||||
|
||||
- name: Delete local temporary directory for certificate generation
|
||||
file:
|
||||
path: "{{ local_sign_dir }}"
|
||||
state: absent
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Create local temporary directory for certificate generation
|
||||
file:
|
||||
path: "{{ local_sign_dir }}"
|
||||
state: directory
|
||||
mode: 0755
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Remove certificates if they already exist
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: absent
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
with_items:
|
||||
- "{{ cert_key_path.local }}"
|
||||
- "{{ cert_csr_path }}"
|
||||
- "{{ cert_single_cert_path }}"
|
||||
- "{{ cert_cert_path.local }}"
|
||||
- "{{ cert_pcks12_cert_path.local }}"
|
||||
|
||||
- name: Use current_host_config.hostname as common name (CN) in certificate generation
|
||||
set_fact:
|
||||
cert_host_fields: "/C={{cert_country}}/ST={{cert_state}}/L={{cert_locality}}/O={{cert_organization}}/OU=sub/CN={{ current_host_config.hostname }}"
|
||||
when: (current_host_config.contains_setup is not defined or current_host_config.contains_setup is defined) and ("mft-cluster-node" not in current_host_config.contains_setup or "fx-cluster" not in current_host_config.contains_setup)
|
||||
|
||||
- name: Use current_host_config.cluster_hostname as common name (CN) in certificate generation if operating as a MFT cluster node
|
||||
set_fact:
|
||||
cert_host_fields: "/C={{cert_country}}/ST={{cert_state}}/L={{cert_locality}}/O={{cert_organization}}/OU=sub/CN={{ current_host_config.cluster_hostname }}"
|
||||
when: current_host_config.contains_setup is defined and ("mft-cluster-node" in current_host_config.contains_setup or "fx-cluster" in current_host_config.contains_setup)
|
||||
|
||||
- name: Create local temporary openssl config
|
||||
template:
|
||||
src: openssl.cnf.j2
|
||||
dest: "{{ cert_config_path }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Generate intermediate CSR
|
||||
command: >
|
||||
openssl req
|
||||
-newkey rsa:{{ cert_key_size }}
|
||||
-nodes
|
||||
-keyout {{ cert_inter_key_path }}
|
||||
-out {{ cert_inter_csr_path }}
|
||||
-days {{ cert_days_valid }}
|
||||
-subj "{{ cert_inter_fields }}"
|
||||
args:
|
||||
creates: "{{ cert_inter_csr_path }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Generate intermediate certificate
|
||||
command: >
|
||||
openssl x509
|
||||
-extfile {{ cert_config_path }}
|
||||
-extensions v3_intermediate_ca
|
||||
-CA {{ cert_ca_cert_path.local }}
|
||||
-CAkey {{ cert_ca_key_path }}
|
||||
-req -in {{ cert_inter_csr_path }}
|
||||
-out {{ cert_inter_cert_path }}
|
||||
-days {{ cert_days_valid }}
|
||||
-CAcreateserial
|
||||
args:
|
||||
creates: "{{ cert_inter_cert_path }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Generate PKCS12 store
|
||||
command: >
|
||||
keytool -storetype PKCS12 -noprompt
|
||||
-storepass {{ cert_pkcs12_pass }}
|
||||
-importcert -file {{ cert_ca_cert_path.local }}
|
||||
-alias ca
|
||||
-keystore {{ cert_truststore_path.local }}
|
||||
args:
|
||||
creates: "{{ cert_truststore_path.local }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Create host key
|
||||
command: >
|
||||
openssl req -new
|
||||
-newkey rsa:{{ cert_key_size }}
|
||||
-nodes
|
||||
-out {{ cert_csr_path }}
|
||||
-keyout {{ cert_key_path.local }}
|
||||
-days {{ cert_days_valid }}
|
||||
-subj "{{ cert_host_fields }}"
|
||||
-config "{{ cert_config_path }}"
|
||||
-extensions v3_req
|
||||
args:
|
||||
creates: "{{ cert_key_path.local }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Create host certificate
|
||||
command: >
|
||||
openssl x509
|
||||
-CA {{ cert_inter_cert_path }}
|
||||
-CAkey {{ cert_inter_key_path }}
|
||||
-req -in {{ cert_csr_path }}
|
||||
-out {{ cert_single_cert_path }}
|
||||
-days {{ cert_days_valid }}
|
||||
-CAcreateserial
|
||||
-extfile "{{ cert_config_path }}"
|
||||
-extensions v3_req
|
||||
args:
|
||||
creates: "{{ cert_single_cert_path }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Concat certificate chain
|
||||
shell: >
|
||||
cat {{ cert_single_cert_path }}
|
||||
{{ cert_inter_cert_path }}
|
||||
{{ cert_ca_cert_path.local }} > {{ cert_cert_path.local }}
|
||||
args:
|
||||
creates: "{{ cert_cert_path.local }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Export host certificate to PCKS12 format
|
||||
command: >
|
||||
openssl pkcs12 -export
|
||||
-inkey {{ cert_key_path.local }}
|
||||
-in {{ cert_cert_path.local }}
|
||||
-chain -CAfile {{ cert_cert_path.local }}
|
||||
-name {{ ansible_hostname }}
|
||||
-out {{ cert_pcks12_cert_path.local }}
|
||||
-passout pass:{{ cert_pkcs12_pass }}
|
||||
-noiter -nomaciter
|
||||
args:
|
||||
creates: "{{ cert_pcks12_cert_path.local }}"
|
||||
become: false
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Copy certificates to host linux
|
||||
copy:
|
||||
src: "{{ item.local }}"
|
||||
dest: "{{ item.remote }}"
|
||||
owner: "{{ cert_owner }}"
|
||||
group: "{{ cert_group }}"
|
||||
mode: 0600
|
||||
with_items:
|
||||
- "{{ cert_key_path }}"
|
||||
- "{{ cert_cert_path }}"
|
||||
- "{{ cert_pcks12_cert_path }}"
|
||||
- "{{ cert_ca_cert_path }}"
|
||||
- "{{ cert_truststore_path }}"
|
||||
when: ansible_os_family != "Windows" and (fx_version is not defined or fx_version is version('3.0.0', '<'))
|
||||
become: true
|
||||
|
||||
- name: Copy certificates to host linux
|
||||
copy:
|
||||
src: "{{ item.local }}"
|
||||
dest: "{{ item.remote_v3 }}"
|
||||
owner: "{{ cert_owner }}"
|
||||
group: "{{ cert_group }}"
|
||||
mode: 0600
|
||||
with_items:
|
||||
- "{{ cert_key_path }}"
|
||||
- "{{ cert_cert_path }}"
|
||||
- "{{ cert_pcks12_cert_path }}"
|
||||
- "{{ cert_ca_cert_path }}"
|
||||
- "{{ cert_truststore_path }}"
|
||||
when: fx_version is defined and ansible_os_family != "Windows" and fx_version is version('3.0.0', '>=')
|
||||
become: true
|
||||
|
||||
|
||||
- name: Copy certificates to host windows
|
||||
win_copy:
|
||||
src: "{{ item.local }}"
|
||||
dest: "{{ item.remote }}"
|
||||
with_items:
|
||||
- "{{ cert_key_path }}"
|
||||
- "{{ cert_cert_path }}"
|
||||
- "{{ cert_pcks12_cert_path }}"
|
||||
- "{{ cert_ca_cert_path }}"
|
||||
- "{{ cert_truststore_path }}"
|
||||
when: ansible_os_family == "Windows"
|
||||
ignore_errors: true
|
||||
|
||||
- name: Disable selinux
|
||||
command: setenforce 0
|
||||
ignore_errors: true
|
||||
|
||||
|
||||
|
||||
# vim:ft=ansible
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
[ v3_ca ]
|
||||
# Extensions for a typical CA (`man x509v3_config`).
|
||||
subjectKeyIdentifier = hash
|
||||
authorityKeyIdentifier = keyid:always,issuer
|
||||
basicConstraints = critical, CA:true
|
||||
keyUsage = critical, digitalSignature, cRLSign, keyCertSign
|
||||
|
||||
[ v3_intermediate_ca ]
|
||||
# Extensions for a typical intermediate CA (`man x509v3_config`).
|
||||
# pathlen 0: no more intermediates beyond this
|
||||
subjectKeyIdentifier = hash
|
||||
authorityKeyIdentifier = keyid:always,issuer
|
||||
basicConstraints = critical, CA:true, pathlen:0
|
||||
keyUsage = critical, digitalSignature, cRLSign, keyCertSign
|
||||
|
||||
[ req ]
|
||||
distinguished_name = req_distinguished_name
|
||||
req_extensions = v3_req
|
||||
prompt = no
|
||||
|
||||
[ req_distinguished_name ]
|
||||
CN = {{ current_host_config.hostname }}
|
||||
|
||||
[ v3_req ]
|
||||
#basicConstraints = CA:FALSE
|
||||
keyUsage = critical, digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = serverAuth, clientAuth
|
||||
subjectAltName = DNS:{{ current_host_config.hostname }}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
---
|
||||
- name: Get Share List
|
||||
uri:
|
||||
url: https://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1/admin/shares
|
||||
status_code: 200
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
client_cert: "{{ cert_cert_path.remote }}"
|
||||
client_key: "{{ cert_key_path.remote }}"
|
||||
validate_certs: false
|
||||
register: shares_response
|
||||
|
||||
- name: Create Local Shares
|
||||
uri:
|
||||
url: https://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1/admin/shares
|
||||
method: POST
|
||||
status_code: 201
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
client_cert: "{{ cert_cert_path.remote }}"
|
||||
client_key: "{{ cert_key_path.remote }}"
|
||||
validate_certs: false
|
||||
body:
|
||||
"name": "{{ item.value.name }}"
|
||||
"local_path": "{{ item.value.local_path }}"
|
||||
"public_uri": "{{ item.value.public_uri }}"
|
||||
"description": "{{ item.value.description }}"
|
||||
"permissions": "{{ item.value.permissions }}"
|
||||
"user_id": "{{ item.value.user_id }}"
|
||||
"public": "{{ item.value.public }}"
|
||||
"file_io_type": "{{ item.value.file_io_type }}"
|
||||
body_format: json
|
||||
vars:
|
||||
existing: "{{ shares_response|json_query('json[*].name') }}"
|
||||
when: item.value.name not in existing
|
||||
with_dict: "{{ local_shares }}"
|
||||
|
||||
# vim:ft=ansible
|
||||
74
infrastructure/ansible/roles/mft-create-nodes/tasks/main.yml
Normal file
74
infrastructure/ansible/roles/mft-create-nodes/tasks/main.yml
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
---
|
||||
|
||||
- name: Get node list, retry on error
|
||||
uri:
|
||||
url: https://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1/admin/nodes
|
||||
status_code: 200
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
client_cert: "{{ cert_cert_path.remote }}"
|
||||
client_key: "{{ cert_key_path.remote }}"
|
||||
validate_certs: false
|
||||
register: nodes_response
|
||||
until: nodes_response.status == 200
|
||||
retries: 20
|
||||
delay: 5
|
||||
become: true
|
||||
|
||||
- name: Create nodes on a host operating as MFT normally
|
||||
uri:
|
||||
url: https://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1/admin/nodes
|
||||
method: POST
|
||||
status_code: 201
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
client_cert: "{{ cert_cert_path.remote }}"
|
||||
client_key: "{{ cert_key_path.remote }}"
|
||||
validate_certs: false
|
||||
body:
|
||||
name: "{{ item.value.name }}"
|
||||
address: "https://{{ item.value.hostname }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1"
|
||||
encryption_policy: "{{item.value.encryption_policy}}"
|
||||
checksum_policy: "{{item.value.checksum_policy}}"
|
||||
body_format: json
|
||||
vars:
|
||||
existing: "{{ nodes_response|json_query('json[*].name') }}"
|
||||
when:
|
||||
- item.value.name not in existing
|
||||
- item.value.contains_setup is defined
|
||||
- "'mft' in item.value.contains_setup"
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
become: true
|
||||
|
||||
- name: Create nodes on a host operating as a clustered MFT
|
||||
uri:
|
||||
url: https://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1/admin/nodes
|
||||
method: POST
|
||||
status_code: 201
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
client_cert: "{{ cert_cert_path.remote }}"
|
||||
client_key: "{{ cert_key_path.remote }}"
|
||||
validate_certs: false
|
||||
body:
|
||||
name: "{{ item.value.name }}"
|
||||
address: "https://{{ item.value.cluster_hostname }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1"
|
||||
encryption_policy: "{{item.value.encryption_policy}}"
|
||||
checksum_policy: "{{item.value.checksum_policy}}"
|
||||
body_format: json
|
||||
|
||||
vars:
|
||||
existing: "{{ nodes_response|json_query('json[*].name') }}"
|
||||
when:
|
||||
- item.value.name not in existing
|
||||
- "'spain' not in existing"
|
||||
- item.value.contains_setup is defined
|
||||
- "'mft-cluster-node' in item.value.contains_setup"
|
||||
- item.value.hostname != 'valencia.tixeltec.de'
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
become: true
|
||||
|
||||
# vim:ft=ansible
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
---
|
||||
|
||||
- name: Get Share List
|
||||
uri:
|
||||
url: https://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.proxy_port }}/transfer-job-manager/v1/admin/shares
|
||||
status_code: 200
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
client_cert: "{{ cert_cert_path.remote }}"
|
||||
client_key: "{{ cert_key_path.remote }}"
|
||||
validate_certs: false
|
||||
register: shares_response
|
||||
until: shares_response.status == 200
|
||||
retries: 10
|
||||
delay: 2
|
||||
become: true
|
||||
|
||||
- name: Create FTP share for relay
|
||||
uri:
|
||||
url: http://{{ ansible_fqdn }}:{{ configs.mft_services.transfer_job_manager.port }}/transfer-job-manager/v1/admin/shares
|
||||
method: POST
|
||||
status_code: 201
|
||||
user: "{{ configs.mft_services.transfer_job_manager.api_user }}"
|
||||
password: "{{ configs.mft_services.transfer_job_manager.api_pass }}"
|
||||
force_basic_auth: yes
|
||||
validate_certs: false
|
||||
body: { "name": "fxrelay",
|
||||
"protocol": "FTP",
|
||||
"host":"{{ current_host_config.fx_server_hostname }}",
|
||||
"port":"21",
|
||||
"username":"fxftp",
|
||||
"password":"verysecret",
|
||||
"directory":"",
|
||||
"description":"ftp2fx",
|
||||
"permissions":"6",
|
||||
"public_uri": "relay_share",
|
||||
"user_id":"1",
|
||||
"public":"true",
|
||||
"relay_share":"true",
|
||||
"relay_server_url":"https://{{ current_host_config.fx_server_hostname }}:{{ configs.fx_services.tixstream_express_job_manager.port }}/fx",
|
||||
"default_relay_recipient_email":"admin@tixeltec.com",
|
||||
"relay_server_username":"demo@tixeltec.com",
|
||||
"relay_server_password":"secret",
|
||||
"file_io_type":"FTP" }
|
||||
body_format: json
|
||||
vars:
|
||||
existing: "{{ shares_response|json_query('json[*].name') }}"
|
||||
when: '"fxrelay" not in existing'
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/bash
|
||||
|
||||
dest=$1
|
||||
test_dir=$2
|
||||
tixstream_user=$4
|
||||
tixstream_group=$5
|
||||
|
||||
if [ "${dest}" == "" ]; then
|
||||
dest="/tmp"
|
||||
fi
|
||||
|
||||
echo "this is a plain text dummy metadata file" >${dest}/metadata.txt
|
||||
|
||||
truncate -s 5g ${dest}/${test_dir}/5g.mxf
|
||||
truncate -s 1g ${dest}/${test_dir}/1g.mxf
|
||||
|
||||
for i in {1..100}; do
|
||||
dd if=/dev/urandom of=/local_testdata/test-source/small_$i.dat bs=1M count=10 status=none
|
||||
done
|
||||
|
||||
chown ${tixstream_user}:${tixstream_group} ${dest}/${test_dir}/*
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
---
|
||||
|
||||
# TODO: this is a duplicate of already existing role: ansible/roles/test-data
|
||||
|
||||
- name: create test data directories
|
||||
file: path={{ local_testdata_dir }}/{{ item }}
|
||||
state=directory
|
||||
owner="{{ localuser }}"
|
||||
group="{{ localgroup }}"
|
||||
mode=0755
|
||||
with_items:
|
||||
- test-source
|
||||
- test-destination
|
||||
become: True
|
||||
|
||||
- name: create recursive test data directories
|
||||
file: path={{ local_testdata_dir }}/{{ item }}
|
||||
state=directory
|
||||
owner="{{ localuser }}"
|
||||
group="{{ localgroup }}"
|
||||
mode=0755
|
||||
with_items:
|
||||
- test-source-recursive
|
||||
become: True
|
||||
|
||||
- name: create subdir 1&2
|
||||
file: path={{ local_testdata_dir }}/test-source-recursive/{{ item }}
|
||||
state=directory
|
||||
owner="{{ localuser }}"
|
||||
group="{{ localgroup }}"
|
||||
mode=0755
|
||||
with_items:
|
||||
- subdir1
|
||||
- subdir2
|
||||
become: True
|
||||
|
||||
- name: create subdir 1_1 & 1_2
|
||||
file: path={{ local_testdata_dir }}/test-source-recursive/subdir1/{{ item }}
|
||||
state=directory
|
||||
owner="{{ localuser }}"
|
||||
group="{{ localgroup }}"
|
||||
mode=0755
|
||||
with_items:
|
||||
- subdir1_1
|
||||
- subdir1_2
|
||||
become: True
|
||||
|
||||
- name: create test files
|
||||
script: create-test-files.sh {{ local_testdata_dir }} test-source test-source-recursive "{{ localuser }}" "{{ localgroup }}"
|
||||
become: True
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
|
||||
- name: change configs dir permissions
|
||||
ansible.builtin.raw: chmod +x /opt/tixel/config/*
|
||||
ignore_errors: true
|
||||
become: true
|
||||
|
||||
- name: Make sure nginx is restarted
|
||||
service: name=nginx state=restarted enabled=True
|
||||
|
||||
- name: Add Debug option to tjm service file
|
||||
ansible.builtin.lineinfile:
|
||||
path: /usr/lib/systemd/system/transfer-job-manager.service
|
||||
regexp: 'ExecStart=/usr/bin/java -Dloader.path=/opt/tixel/drivers/tjm -Dsun.misc.URLClassPath.disableJarChecking=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Djavax.net.ssl.trustStore=/opt/tixel/config/trusted.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.keyStore=/opt/tixel/config/host.p12 -Djavax.net.ssl.keyStorePassword=changeit -Dlogging.config=/opt/tixel/config/tjm-logback-spring-prod.xml -jar /opt/tixel/transfer-job-manager/bin/transfer-job-manager.jar --spring.config.location=classpath:application.properties,file:/opt/tixel/config/transfer-job-manager.properties'
|
||||
line: 'ExecStart=/usr/bin/java -Dloader.path=/opt/tixel/drivers/tjm -Dsun.misc.URLClassPath.disableJarChecking=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Djavax.net.ssl.trustStore=/opt/tixel/config/trusted.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.keyStore=/opt/tixel/config/host.p12 -Djavax.net.ssl.keyStorePassword=changeit -Dlogging.config=/opt/tixel/config/tjm-logback-spring-prod.xml -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 -jar /opt/tixel/transfer-job-manager/bin/transfer-job-manager.jar --spring.config.location=classpath:application.properties,file:/opt/tixel/config/transfer-job-manager.properties'
|
||||
|
||||
- name: Add Debug option to am service file
|
||||
ansible.builtin.lineinfile:
|
||||
path: /usr/lib/systemd/system/access-manager.service
|
||||
regexp: 'ExecStart=/usr/bin/java -Dsun.misc.URLClassPath.disableJarChecking=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Djavax.net.ssl.trustStore=/opt/tixel/config/trusted.p12 -Djavax.net.ssl.trustStorePassword=changeit -jar /opt/tixel/access-manager/bin/access-manager.jar --spring.config.location=classpath:application.properties,file:/opt/tixel/config/access-manager.properties'
|
||||
line: 'ExecStart=/usr/bin/java -Dsun.misc.URLClassPath.disableJarChecking=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Djavax.net.ssl.trustStore=/opt/tixel/config/trusted.p12 -Djavax.net.ssl.trustStorePassword=changeit -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5006 -jar /opt/tixel/access-manager/bin/access-manager.jar --spring.config.location=classpath:application.properties,file:/opt/tixel/config/access-manager.properties'
|
||||
|
||||
- name: Add Debug option to tcc service file
|
||||
ansible.builtin.lineinfile:
|
||||
path: /usr/lib/systemd/system/tixel-control-center.service
|
||||
regexp: 'ExecStart=/usr/bin/java -Dsun.misc.URLClassPath.disableJarChecking=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Djavax.net.ssl.trustStore=/opt/tixel/config/trusted.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.keyStore=/opt/tixel/config/host.p12 -Djavax.net.ssl.keyStorePassword=changeit -jar /opt/tixel/tixel-control-center/bin/tixel-control-center.jar --spring.config.location=classpath:application.properties,file:/opt/tixel/config/tixel-control-center.properties'
|
||||
line: 'ExecStart=/usr/bin/java -Dsun.misc.URLClassPath.disableJarChecking=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Djavax.net.ssl.trustStore=/opt/tixel/config/trusted.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.keyStore=/opt/tixel/config/host.p12 -Djavax.net.ssl.keyStorePassword=changeit -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5007 -jar /opt/tixel/tixel-control-center/bin/tixel-control-center.jar --spring.config.location=classpath:application.properties,file:/opt/tixel/config/tixel-control-center.properties'
|
||||
|
||||
- name: reload systemctl
|
||||
command: systemctl daemon-reload
|
||||
|
||||
- name: Installiere Java 17 (für TIXstream Runtime)
|
||||
apt:
|
||||
name: openjdk-21-jre-headless
|
||||
state: present
|
||||
- name: Setze Java 17 als Standard (alternatives)
|
||||
community.general.alternatives:
|
||||
name: java
|
||||
path: /usr/lib/jvm/java-21-openjdk-amd64/bin/java
|
||||
link: /usr/bin/java
|
||||
|
||||
- name: Restart MFT services
|
||||
service: name={{ item.value.name }} state=restarted
|
||||
with_dict: "{{ configs.mft_services }}"
|
||||
|
||||
- name: Wait for tixel_control_center to be reachable
|
||||
wait_for: host="{{ ansible_fqdn }}" port="{{ configs.mft_services.tixel_control_center.port }}" delay=1 timeout=200
|
||||
|
||||
- name: Wait for transfer_job_manager to be reachable
|
||||
wait_for: host="{{ ansible_fqdn }}" port="{{ configs.mft_services.transfer_job_manager.port }}" delay=1 timeout=200
|
||||
|
||||
# vim:ft=ansible
|
||||
134
infrastructure/ansible/roles/mft-setup-mariadb/tasks/main.yml
Normal file
134
infrastructure/ansible/roles/mft-setup-mariadb/tasks/main.yml
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
---
|
||||
|
||||
- name: Set database server to localhost if undefined
|
||||
set_fact:
|
||||
database_server_ip: "127.0.0.1"
|
||||
when: current_host_config.mariadb_server_ip is undefined
|
||||
|
||||
- name: Set database server to localhost if defined
|
||||
set_fact:
|
||||
database_server_ip: "{{current_host_config.mariadb_server_ip}}"
|
||||
when: current_host_config.mariadb_server_ip is defined
|
||||
|
||||
- name: Set transfer job manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.datasource.platform"
|
||||
line: "spring.datasource.platform=mysql"
|
||||
- option: "datasource.mysql.username"
|
||||
line: "datasource.mysql.username=tixel"
|
||||
- option: "datasource.mysql.password"
|
||||
line: "datasource.mysql.password=tixel"
|
||||
- option: "datasource.mysql.url"
|
||||
line: "datasource.mysql.url=jdbc\\:mysql\\://{{ database_server_ip }}\\:3306/transfer_job_manager?useSSL=false"
|
||||
when: "mft_version is version('2.2.0', '<=')"
|
||||
|
||||
- name: Set access manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.access_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.datasource.platform"
|
||||
line: "spring.datasource.platform=mysql"
|
||||
- option: "datasource.mysql.username"
|
||||
line: "datasource.mysql.username=tixel"
|
||||
- option: "datasource.mysql.password"
|
||||
line: "datasource.mysql.password=tixel"
|
||||
- option: "datasource.mysql.url"
|
||||
line: "datasource.mysql.url=jdbc\\:mysql\\://{{ database_server_ip }}\\:3306/access_manager?useSSL=false"
|
||||
when: "mft_version is version('2.2.0', '<=')"
|
||||
|
||||
- name: Set transfer job manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.datasource.platform"
|
||||
line: "spring.datasource.platform=mysql"
|
||||
- option: "datasource.mysql.username"
|
||||
line: "datasource.mysql.username=tixel"
|
||||
- option: "datasource.mysql.password"
|
||||
line: "datasource.mysql.password=tixel"
|
||||
- option: "datasource.mysql.url"
|
||||
line: "datasource.mysql.url=jdbc\\:mysql\\://{{ database_server_ip }}\\:3306/transfer_job_manager"
|
||||
when: "mft_version is version('2.2.0', '<=')"
|
||||
|
||||
- name: Set access manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.access_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.datasource.platform"
|
||||
line: "spring.datasource.platform=mysql"
|
||||
- option: "datasource.mysql.username"
|
||||
line: "datasource.mysql.username=tixel"
|
||||
- option: "datasource.mysql.password"
|
||||
line: "datasource.mysql.password=tixel"
|
||||
- option: "datasource.mysql.url"
|
||||
line: "datasource.mysql.url=jdbc\\:mysql\\://{{ database_server_ip }}\\:3306/access_manager"
|
||||
when: "mft_version is version('2.2.0', '<=')"
|
||||
|
||||
- name: Set transfer job manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.sql.init.platform"
|
||||
line: "spring.sql.init.platform=mysql"
|
||||
- option: "datasource.mysql.username"
|
||||
line: "datasource.mysql.username=tixel"
|
||||
- option: "datasource.mysql.password"
|
||||
line: "datasource.mysql.password=tixel"
|
||||
- option: "datasource.mysql.url"
|
||||
line: "datasource.mysql.url=jdbc\\:mysql\\://{{ database_server_ip }}\\:3306/transfer_job_manager?useSSL=false"
|
||||
when: mft_version is version('2.3.0', '>=')
|
||||
|
||||
- name: Set access manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.access_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.sql.init.platform"
|
||||
line: "spring.sql.init.platform=mysql"
|
||||
- option: "datasource.mysql.username"
|
||||
line: "datasource.mysql.username=tixel"
|
||||
- option: "datasource.mysql.password"
|
||||
line: "datasource.mysql.password=tixel"
|
||||
- option: "datasource.mysql.url"
|
||||
line: "datasource.mysql.url=jdbc\\:mysql\\://{{ database_server_ip }}\\:3306/access_manager"
|
||||
when: mft_version is version('2.3.0', '>=') and (ansible_distribution_major_version != "24" or ansible_distribution != "Rocky")
|
||||
|
||||
- name: Set access manager database properties
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.access_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items:
|
||||
- option: "spring.sql.init.platform"
|
||||
line: "spring.sql.init.platform=mariadb"
|
||||
- option: "datasource.mariadb.username"
|
||||
line: "datasource.mariadb.username=tixel"
|
||||
- option: "datasource.mariadb.password"
|
||||
line: "datasource.mariadb.password=tixel"
|
||||
- option: "datasource.mariadb.url"
|
||||
line: "datasource.mariadb.url=jdbc\\:mariadb\\://{{ database_server_ip }}\\:3306/access_manager"
|
||||
when: mft_version is version('2.3.0', '>=') and (ansible_distribution_major_version == "24" or ansible_distribution == "Rocky")
|
||||
|
||||
|
||||
|
||||
# vim:ft=ansible
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
- name: Restart nginx with config check
|
||||
debug: msg="checking nginx config before restart"
|
||||
changed_when: True
|
||||
notify:
|
||||
- Check nginx configuration
|
||||
- Restart nginx
|
||||
|
||||
- name: Reload nginx with config check
|
||||
debug: msg="checking nginx config before reload"
|
||||
changed_when: True
|
||||
notify:
|
||||
- Check nginx configuration
|
||||
- Reload nginx
|
||||
|
||||
- name: Check nginx configuration
|
||||
command: "nginx -t"
|
||||
register: result
|
||||
changed_when: "result.rc != 0"
|
||||
check_mode: no
|
||||
|
||||
- name: Restart nginx
|
||||
service: name=nginx state=restarted
|
||||
|
||||
- name: Reload nginx
|
||||
service: name=nginx state=reloaded
|
||||
|
||||
# vim:ft=ansible
|
||||
86
infrastructure/ansible/roles/mft-setup-nginx/tasks/main.yml
Normal file
86
infrastructure/ansible/roles/mft-setup-nginx/tasks/main.yml
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
---
|
||||
|
||||
- sefcontext:
|
||||
target: "{{ configs.mft_services.nginx.log_dir }}(/.*)?"
|
||||
setype: httpd_sys_rw_content_t
|
||||
state: present
|
||||
when: ansible_distribution == 'CentOS'
|
||||
|
||||
- name: Apply SELinux context changes
|
||||
command: restorecon -R -v "{{ configs.mft_services.nginx.log_dir }}"
|
||||
when: ansible_distribution == 'CentOS'
|
||||
|
||||
- name: Allow nginx to access cifs shares
|
||||
seboolean: name=httpd_use_cifs state=yes persistent=yes
|
||||
when: ansible_distribution == 'CentOS'
|
||||
|
||||
- name: Create nginx group for ubuntu
|
||||
group:
|
||||
name: "{{ configs.mft_services.nginx.user }}"
|
||||
state: present
|
||||
when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian'
|
||||
|
||||
- name: Create nginx user for ubuntu
|
||||
user:
|
||||
name: "{{ configs.mft_services.nginx.user }}"
|
||||
groups: "{{ configs.mft_services.nginx.group }}"
|
||||
state: present
|
||||
createhome: no
|
||||
when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian'
|
||||
|
||||
- name: Find nginx config
|
||||
find:
|
||||
paths: "{{ remote_deployment_dir }}"
|
||||
patterns: nginx_https.conf
|
||||
file_type: file
|
||||
recurse: yes
|
||||
register: found_nginx_config
|
||||
|
||||
- name: Copy nginx configuration
|
||||
command: cp {{ found_nginx_config.files[0].path }} {{ configs.mft_services.nginx.config }}
|
||||
notify:
|
||||
- Restart nginx with config check
|
||||
|
||||
- name: Open port 60011 in nginx config
|
||||
lineinfile:
|
||||
path: "{{ configs.mft_services.nginx.config }}"
|
||||
insertbefore: '[ \t]* access_log /var/log/nginx/access.log;'
|
||||
line: ' listen {{ configs.mft_services.access_manager.proxy_port }};'
|
||||
become: true
|
||||
|
||||
- name: Add Access Manager forwarding in nginx configuration
|
||||
blockinfile:
|
||||
path: "{{ configs.mft_services.nginx.config }}"
|
||||
insertafter: '[ \t]* access_log /var/log/nginx/access.log;'
|
||||
block: |
|
||||
location /access-manager/oauth/token {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/oauth/token;
|
||||
}
|
||||
location /access-manager/v1/admin/clients {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/v1/admin/clients;
|
||||
}
|
||||
location /access-manager/v1/greeting {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/v1/greeting;
|
||||
}
|
||||
location /access-manager/v1/admin/roles {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/v1/admin/roles;
|
||||
}
|
||||
location /access-manager/v1/admin/users {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/v1/admin/users;
|
||||
}
|
||||
location /access-manager/v1/me {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/v1/me;
|
||||
}
|
||||
location /access-manager/v1/admin/user-role-membership {
|
||||
proxy_pass http://127.0.0.1:9001/access-manager/v1/admin/user-role-membership;
|
||||
}
|
||||
become: true
|
||||
|
||||
- name: Set nginx config ownership
|
||||
file:
|
||||
path: "{{ configs.mft_services.nginx.config }}"
|
||||
owner: "{{ configs.mft_services.nginx.user }}"
|
||||
group: "{{ configs.mft_services.nginx.group }}"
|
||||
mode: "0660"
|
||||
|
||||
# vim:ft=ansible
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
---
|
||||
|
||||
- name: Enable email notifications
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.transfer-job-manager.email-notification-enabled.*$'
|
||||
line: "custom.transfer-job-manager.email-notification-enabled=true"
|
||||
state: present
|
||||
|
||||
- name: Set smtp host
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.host.*$'
|
||||
line: "spring.mail.host={{ current_host_config.smtplog_server_ip | default(current_host_config.ip) }}"
|
||||
state: present
|
||||
|
||||
- name: Set smtp port
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.port.*$'
|
||||
line: "spring.mail.port={{ smtplog_port }}"
|
||||
state: present
|
||||
|
||||
- name: Set smtp user
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.username.*$'
|
||||
line: "spring.mail.username="
|
||||
state: present
|
||||
|
||||
- name: Set smtp password
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.password.*$'
|
||||
line: "spring.mail.password="
|
||||
state: present
|
||||
|
||||
- name: Set smtp authentication
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.properties.mail.smtp.auth.*$'
|
||||
line: "spring.mail.properties.mail.smtp.auth=false"
|
||||
state: present
|
||||
|
||||
- name: Set smtp ssl trust
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.properties.mail.smtp.ssl.trust.*$'
|
||||
line: "spring.mail.properties.mail.smtp.ssl.trust=*"
|
||||
state: present
|
||||
|
||||
- name: Set smtp starttls
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.properties.mail.smtp.starttls.enable.*$'
|
||||
line: "spring.mail.properties.mail.smtp.starttls.enable=false"
|
||||
state: present
|
||||
|
||||
- name: Set smtp tls
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.tls.*$'
|
||||
line: "spring.mail.tls=false"
|
||||
state: present
|
||||
|
||||
- name: Set smtp protocol
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.protocol.*$'
|
||||
line: "spring.mail.protocol=smtp"
|
||||
state: present
|
||||
|
||||
- name: Set mail encoding
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^spring.mail.defaultEncoding.*$'
|
||||
line: "spring.mail.defaultEncoding=UTF-8"
|
||||
state: present
|
||||
|
||||
# vim:ft=ansible
|
||||
16
infrastructure/ansible/roles/mft-tls-enable/tasks/main.yml
Normal file
16
infrastructure/ansible/roles/mft-tls-enable/tasks/main.yml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
---
|
||||
- name: Set TIXstream internal-client-tlsverify = 1
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.tixstream.config }}"
|
||||
regexp: '^internal-client-tlsverify.*$'
|
||||
line: "internal-client-tlsverify = 1"
|
||||
state: present
|
||||
|
||||
- name: Set TIXstream receiver port in Transfer-Job-Manager config
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.tixstream.internal.port.*$'
|
||||
line: "custom.tixstream.internal.port={{ configs.mft_services.tixstream.proxy_port }}"
|
||||
state: present
|
||||
|
||||
# vim:ft=ansible
|
||||
128
infrastructure/ansible/roles/mft/tasks/main.yml
Normal file
128
infrastructure/ansible/roles/mft/tasks/main.yml
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
---
|
||||
|
||||
- name: Create tixstream group
|
||||
group:
|
||||
name: "{{ smbgroup }}"
|
||||
state: present
|
||||
|
||||
- name: create tixstream user
|
||||
user:
|
||||
name: "{{ smbuser }}"
|
||||
shell: /bin/bash
|
||||
groups: "{{ smbgroup }}"
|
||||
append: yes
|
||||
|
||||
- name: create deployment directory
|
||||
file: path={{ remote_deployment_dir }} state=directory
|
||||
|
||||
- include_tasks: mft-ubuntu-24.yml
|
||||
when: ansible_distribution == "Ubuntu" and ansible_facts['distribution_version'] == "24.04"
|
||||
|
||||
- name: Print the package name
|
||||
debug:
|
||||
msg: "The box is {{ package_to_install }}"
|
||||
|
||||
- unarchive:
|
||||
src: "{{ package_to_install.path }}"
|
||||
dest: "{{ remote_deployment_dir }}"
|
||||
when: oldpackage is undefined and newpackage is undefined
|
||||
|
||||
- unarchive:
|
||||
src: "{{ package_to_install }}"
|
||||
dest: "{{ remote_deployment_dir }}"
|
||||
when: oldpackage is defined or newpackage is defined
|
||||
|
||||
- name: List unarchived MFT bundle directory
|
||||
find:
|
||||
paths: "{{ remote_deployment_dir }}"
|
||||
patterns: "tixstream-mft-ubuntu*"
|
||||
file_type: directory
|
||||
register: found_directories
|
||||
|
||||
- name: Get latest unarchived MFT bundle directory
|
||||
set_fact:
|
||||
package_dir: "{{ found_directories.files | sort(attribute='mtime',reverse=true) | first }}"
|
||||
|
||||
- name: Check if an old version should be deployed (install script has different name)
|
||||
stat: path={{ package_dir.path }}/install_mft_bundle.sh
|
||||
register: old_install_script
|
||||
|
||||
- name: Set the correct name of the install script
|
||||
set_fact:
|
||||
install_script: "{{'install_mft_bundle.sh' if old_install_script.stat.exists else 'install.sh'}}"
|
||||
|
||||
- name: Execute MFT install script
|
||||
command: "{{ package_dir.path }}/{{ install_script }} --assumeyes -u {{ smbuser }} -g {{ smbgroup }}"
|
||||
|
||||
- name: Set name to the nodes name in transfer-job-manager config
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.transfer-job-manager.name.*$'
|
||||
line: "custom.transfer-job-manager.name={{ item.value.name }}"
|
||||
state: present
|
||||
when: item.value.hostname == ansible_fqdn and item.value.contains_setup is defined and "mft" in item.value.contains_setup
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
|
||||
- name: Set hostname to the nodes hostname in transfer-job-manager config for a normal MFT node
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.transfer-job-manager.hostname.*$'
|
||||
line: "custom.transfer-job-manager.hostname={{ item.value.hostname }}"
|
||||
state: present
|
||||
when: item.value.hostname == ansible_fqdn and item.value.contains_setup is defined and "mft" in item.value.contains_setup
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
|
||||
- name: Set hostname to the nodes hostname in transfer-job-manager config for a clustered MFT node
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.transfer-job-manager.hostname.*$'
|
||||
line: "custom.transfer-job-manager.hostname={{ item.value.cluster_hostname }}"
|
||||
state: present
|
||||
when: item.value.hostname == ansible_fqdn and item.value.contains_setup is defined and "mft-cluster-node" in item.value.contains_setup
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
|
||||
- name: Set transfer-job-manager job scheduling options in the config
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items: "{{ configs.mft_services.transfer_job_manager.scheduler_properties }}"
|
||||
|
||||
- name: Set transfer-job-manager sending max-datarate options in the config
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.transfer-job-manager.default-sending-data-rate'
|
||||
line: "custom.transfer-job-manager.default-sending-data-rate=10000000"
|
||||
state: present
|
||||
|
||||
- name: Set transfer-job-manager receiving max-datarate options in the config
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^custom.transfer-job-manager.default-receiving-data-rate'
|
||||
line: "custom.transfer-job-manager.default-receiving-data-rate=10000000"
|
||||
state: present
|
||||
|
||||
- name: Enable transfer-job-manager watch folder feature
|
||||
lineinfile:
|
||||
dest: "{{ configs.mft_services.transfer_job_manager.config }}"
|
||||
regexp: '^{{ item.option }}.*$'
|
||||
line: "{{ item.line }}"
|
||||
state: present
|
||||
with_items: "{{ configs.mft_services.transfer_job_manager.watchfolder_properties }}"
|
||||
|
||||
# Use command module because service module fails to just 'enable' the services
|
||||
- name: Enable MFT services to start on boot for centos
|
||||
command: chkconfig {{ item.value.name }} on
|
||||
with_dict: "{{ configs.mft_services }}"
|
||||
when: (ansible_distribution == 'CentOS')
|
||||
|
||||
- name: Enable MFT services to start on boot for Ubuntu
|
||||
service:
|
||||
name: "{{ item.value.name }}"
|
||||
enabled: yes
|
||||
with_dict: "{{ configs.mft_services }}"
|
||||
when: (ansible_distribution == 'Ubuntu')
|
||||
|
||||
|
||||
# vim:ft=ansible
|
||||
102
infrastructure/ansible/roles/mft/tasks/mft-ubuntu-24.yml
Normal file
102
infrastructure/ansible/roles/mft/tasks/mft-ubuntu-24.yml
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
---
|
||||
|
||||
- name: Install package dependencies for MFT on Ubuntu Java 8
|
||||
apt:
|
||||
name: openjdk-8-jdk
|
||||
state: present
|
||||
update_cache: yes
|
||||
become: true
|
||||
when: '"java8" in current_host_config.contains_setup'
|
||||
|
||||
- name: Install package dependencies for MFT on Ubuntu Java 11
|
||||
apt:
|
||||
name: openjdk-11-jdk
|
||||
state: present
|
||||
update_cache: yes
|
||||
become: true
|
||||
when: '"java11" in current_host_config.contains_setup'
|
||||
|
||||
- name: Install package dependencies for MFT on Ubuntu Java 21
|
||||
apt:
|
||||
name: openjdk-21-jre-headless
|
||||
state: present
|
||||
update_cache: yes
|
||||
become: true
|
||||
when: '"java21" in current_host_config.contains_setup'
|
||||
|
||||
#- name: Get package dependencies for MFT on CentOS Java21
|
||||
# shell: "curl -O https://download.java.net/java/GA/jdk21.0.1/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-21.0.1_linux-x64_bin.tar.gz && sudo tar xvf openjdk-21.0.1_linux-x64_bin.tar.gz && sudo mv jdk-21.0.1 /opt/"
|
||||
# args:
|
||||
# chdir: /home/vagrant
|
||||
#become: true
|
||||
#when: '"java21" in current_host_config.contains_setup'
|
||||
|
||||
#- name: Get package dependencies for MFT on CentOS Java21
|
||||
#shell: "sudo update-alternatives --install /usr/bin/java java /opt/jdk-21.0.1/bin/java 99"
|
||||
#args:
|
||||
# chdir: /opt/
|
||||
#become: true
|
||||
#when: '"java21" in current_host_config.contains_setup'
|
||||
|
||||
- name: List MFT packages
|
||||
local_action: find paths="{{ configs.deployment_dir }}" patterns="tixstream-mft-{{ ansible_distribution|lower }}-24*.tgz" file_type=file
|
||||
become: false
|
||||
register: found_packages
|
||||
|
||||
- name: Get latest package
|
||||
set_fact:
|
||||
package_to_install: "{{ found_packages.files | sort(attribute='mtime',reverse=true) | first }}"
|
||||
when: package_name is undefined or package_name == ""
|
||||
|
||||
- name: Set package name
|
||||
set_fact:
|
||||
package_to_install: "{{ configs.deployment_dir }}/{{ package_name }}"
|
||||
when: package_name is defined and package_name != ""
|
||||
|
||||
- name: Get MFT version
|
||||
set_fact:
|
||||
mft_version: "{{ package_to_install.path | basename | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+(?:\\.[0-9]+)?') }}"
|
||||
|
||||
- name: Check if Codemeter is installed
|
||||
shell: "cmu -v"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: codemeter_installed
|
||||
ignore_errors: true
|
||||
when: mft_version is version('2.1.0', '>=')
|
||||
|
||||
- name: Codemeter packages local
|
||||
local_action: find paths="{{ configs.deployment_dir }}" patterns="codemeter*.deb" file_type=file
|
||||
become: false
|
||||
register: codemeter_packages
|
||||
when: mft_version is version('2.1.0', '>=')
|
||||
|
||||
- name: Set deb package
|
||||
set_fact:
|
||||
codemeter_deb_path: "{{ codemeter_packages.files | sort(attribute='mtime',reverse=true) | first }}"
|
||||
when: mft_version is version('2.1.0', '>=')
|
||||
|
||||
- name: Copy Codemeter deb to VM
|
||||
copy:
|
||||
src: "{{ codemeter_deb_path.path }}"
|
||||
dest: "{{ remote_deployment_dir }}"
|
||||
when: mft_version is version('2.1.0', '>=')
|
||||
|
||||
- name: Codemeter packages remote
|
||||
find:
|
||||
paths: "{{ remote_deployment_dir }}"
|
||||
patterns: "codemeter*.deb"
|
||||
file_type: file
|
||||
register: remote_codemeter_path
|
||||
when: mft_version is version('2.1.0', '>=')
|
||||
|
||||
- name: Set deb package remote
|
||||
set_fact:
|
||||
codemeter_remote_deb: "{{ remote_codemeter_path.files | sort(attribute='mtime',reverse=true) | first }}"
|
||||
when: mft_version is version('2.1.0', '>=')
|
||||
|
||||
- name: Install Codemeter
|
||||
apt:
|
||||
deb: "{{ codemeter_remote_deb.path }}"
|
||||
state: present
|
||||
when: codemeter_installed.rc is defined and codemeter_installed.rc != 0 and mft_version is version('2.1.0', '>=')
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
- name: Restart Watch-Tool
|
||||
systemd:
|
||||
name: watch-tool
|
||||
state: restarted
|
||||
daemon_reload: yes
|
||||
109
infrastructure/ansible/roles/monitoring_baseline/tasks/main.yml
Normal file
109
infrastructure/ansible/roles/monitoring_baseline/tasks/main.yml
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
---
|
||||
- name: Kopiere Journal Dumper Script
|
||||
template:
|
||||
src: journal_dumper.sh.j2
|
||||
dest: /usr/local/bin/journal_dumper.sh
|
||||
mode: "0755"
|
||||
|
||||
- name: Deploye Systemd Services für Monitoring
|
||||
copy:
|
||||
dest: "/etc/systemd/system/{{ item }}.service"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Thesis {{ item }} Service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/{{ item }}.sh
|
||||
Restart=always
|
||||
User=root
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
loop:
|
||||
- journal_dumper
|
||||
|
||||
- name: Starte und aktiviere Monitoring Services
|
||||
systemd:
|
||||
name: "{{ item }}"
|
||||
state: started
|
||||
enabled: yes
|
||||
daemon_reload: yes
|
||||
loop:
|
||||
- journal_dumper
|
||||
|
||||
- name: Erstelle Temp-Verzeichnis
|
||||
tempfile:
|
||||
state: directory
|
||||
register: tmpdir
|
||||
|
||||
- name: Download tar.gz
|
||||
get_url:
|
||||
url: "https://codeberg.org/Pata1704/metrics-collector/releases/download/v1.1.0/metrics-collector_Linux_x86_64.tar.gz"
|
||||
dest: "{{ tmpdir.path }}/metrics-collector.tar.gz"
|
||||
retries: 3
|
||||
delay: 5
|
||||
|
||||
- name: Entpacke Binary
|
||||
unarchive:
|
||||
src: "{{ tmpdir.path }}/metrics-collector.tar.gz"
|
||||
dest: "{{ tmpdir.path }}"
|
||||
remote_src: yes
|
||||
|
||||
- name: Installiere Binary
|
||||
copy:
|
||||
src: "{{ tmpdir.path }}/metrics-collector"
|
||||
dest: /usr/local/bin/metrics-collector
|
||||
mode: "0755"
|
||||
remote_src: yes
|
||||
|
||||
- name: Cleanup Temp
|
||||
file:
|
||||
path: "{{ tmpdir.path }}"
|
||||
state: absent
|
||||
|
||||
- name: Deploy systemd Service
|
||||
copy:
|
||||
dest: /etc/systemd/system/metrics-collector.service
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Thesis Metrics Collector
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/metrics-collector
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
TimeoutStopSec=60
|
||||
KillMode=mixed
|
||||
KillSignal=SIGTERM
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
register: service_file
|
||||
|
||||
- name: Reload systemd
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
when: service_file.changed
|
||||
|
||||
- name: Enable und Start Service
|
||||
systemd:
|
||||
name: metrics-collector
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
- name: Prüfe Service-Status
|
||||
command: systemctl is-active metrics-collector
|
||||
register: service_check
|
||||
changed_when: false
|
||||
failed_when: service_check.stdout != "active"
|
||||
|
||||
- name: Zeige Erfolg
|
||||
debug:
|
||||
msg: "Metrics Collector erfolgreich deployed!"
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
TARGET="/local_testdata/test-destination"
|
||||
|
||||
echo "Starte Blackhole auf $TARGET"
|
||||
|
||||
while true; do
|
||||
find "$TARGET" -type f -mmin +0.5 -delete
|
||||
sleep 10
|
||||
done
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
export:
|
||||
enabled: true
|
||||
batch_size: 100
|
||||
export_interval: "30s"
|
||||
retry_attempts: 5
|
||||
retry_backoff: "10s"
|
||||
health_check_interval: "60s"
|
||||
|
||||
localstorage:
|
||||
enabled: true
|
||||
db_path: "./watch.db"
|
||||
rotation:
|
||||
max_sizes_bytes: 100 * 1024 * 1024
|
||||
max_age_hours: 24
|
||||
max_files: 3
|
||||
check_interval_minuntes: 5
|
||||
archive_dir: ""
|
||||
|
||||
elasticsearch:
|
||||
enabled: false
|
||||
url: "http://10.0.0.99:9200"
|
||||
index: "watch"
|
||||
username: "your-configured-user"
|
||||
password: "your-super-secret-password"
|
||||
api_key: "your-api-key"
|
||||
timeout: 30
|
||||
|
||||
web_service:
|
||||
enabled: true
|
||||
host: "0.0.0.0"
|
||||
port: 9090
|
||||
|
||||
system_metrics:
|
||||
enabled: true
|
||||
collect_cpu: true
|
||||
collect_memory: true
|
||||
collect_disk: true
|
||||
collect_network: true
|
||||
disk_paths:
|
||||
- "/"
|
||||
- "/var"
|
||||
- "/home"
|
||||
network_interfaces:
|
||||
- "ens6"
|
||||
collect_network_connections: true
|
||||
collect_load_average: true
|
||||
collect_tcp_stats: true
|
||||
collect_filehandles: true
|
||||
collect_disk_io: true
|
||||
collect_network_latency: true
|
||||
collect_bandwidth_usage: true
|
||||
transfer_ports: 60003
|
||||
latency_test_hosts: "www.google.de"
|
||||
|
||||
poll_interval_seconds: 30
|
||||
patterns_file: "./configs/patterns.yaml"
|
||||
|
||||
logging:
|
||||
level: "info"
|
||||
file_path: "/var/log/system-monitor.log"
|
||||
|
||||
drain3:
|
||||
enabled: true
|
||||
state_dir: "./drain3_states"
|
||||
depth: 4
|
||||
sim_th: 0.4
|
||||
max_children: 100
|
||||
max_clusters: 1000
|
||||
save_interval: 60
|
||||
|
||||
services:
|
||||
- name: "nginx"
|
||||
service: "nginx.service"
|
||||
enabled: true
|
||||
since_time: ""
|
||||
priority: "info"
|
||||
|
||||
tools:
|
||||
- name: "nginx-access"
|
||||
log_file: "/var/log/nginx/access.log"
|
||||
enabled: true
|
||||
buffer_size: 200
|
||||
format:
|
||||
name: "nginx_combined"
|
||||
pattern: '^(?P<client_ip>\S+) \S+ \S+ \[(?P<timestamp>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+) (?P<protocol>\S+)" (?P<status>\d+) (?P<body_bytes>\d+) "(?P<referer>[^"]*)" "(?P<user_agent>[^"]*)"'
|
||||
fields:
|
||||
client_ip: "remote_addr"
|
||||
timestamp: "time_local"
|
||||
method: "request_method"
|
||||
path: "request_uri"
|
||||
protocol: "server_protocol"
|
||||
status: "status"
|
||||
body_bytes: "body_bytes_sent"
|
||||
referer: "http_referer"
|
||||
user_agent: "http_user_agent"
|
||||
|
||||
- name: "nginx-error"
|
||||
log_file: "/var/log/nginx/error.log"
|
||||
enabled: true
|
||||
buffer_size: 100
|
||||
format:
|
||||
name: "nginx_error"
|
||||
pattern: '^(?P<timestamp>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<level>\w+)\] (?P<pid>\d+)#(?P<tid>\d+): (?P<message>.*)'
|
||||
fields:
|
||||
timestamp: "time"
|
||||
level: "log_level"
|
||||
pid: "process_id"
|
||||
tid: "thread_id"
|
||||
message: "error_message"
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
# Verzeichnis, in dem TIXstream die Dateien ablegt (anpassen!)
|
||||
INCOMING_DIR="/data/tixel/incoming"
|
||||
|
||||
echo "Starte Blackhole-Service für $INCOMING_DIR..."
|
||||
|
||||
while true; do
|
||||
# Lösche alle Dateien, die älter als 1 Minute sind (damit wir laufende Transfers nicht killen)
|
||||
# Oder radikaler: Sobald der Transfer fertig ist (Tixstream benennt temporäre Dateien oft um).
|
||||
# Wir nehmen hier sicherheitshalber Dateien, auf die seit 10s nicht zugegriffen wurde.
|
||||
|
||||
find "$INCOMING_DIR" -type f -mmin +0.1 -delete
|
||||
|
||||
sleep 10
|
||||
done
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
# Speichert Logs der relevanten Services (TJM und Tixstream Engine)
|
||||
# -f: Follow (Live)
|
||||
# -u: Unit filter
|
||||
# --no-pager: Wichtig für Background-Prozesse
|
||||
# -o json: Speichert strukturiertes JSON (perfekt für deine ML-Pipeline!)
|
||||
|
||||
LOGfile="/var/log/thesis_tixstream_logs.json"
|
||||
|
||||
# Wir loggen Tixstream Engine UND den Job Manager
|
||||
exec journalctl -u tixstream -u transfer-job-manager -f -o json --no-pager >> "$LOGfile"
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
patterns:
|
||||
# ===========================================================================
|
||||
# Common / Shared Patterns
|
||||
# ===========================================================================
|
||||
common:
|
||||
extractors:
|
||||
- name: "syslog_header"
|
||||
regex: '^(\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<hostname>[^\s]+) (?P<process_info>[^:]+):\s*(?P<message_rest>.*)$'
|
||||
fields:
|
||||
syslog_timestamp: "time:Jan 02 15:04:05"
|
||||
hostname: "string"
|
||||
process_info: "string"
|
||||
message_rest: "string"
|
||||
|
||||
- name: "timestamp_rfc3339"
|
||||
regex: '(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?)'
|
||||
fields:
|
||||
timestamp: "time:2006-01-02T15:04:05.000000Z"
|
||||
|
||||
# ===========================================================================
|
||||
# TIXstream Service
|
||||
# Deckt ab: tsServicePattern, tsTransferIDPattern, tsDetailPattern1-4
|
||||
# ===========================================================================
|
||||
tixstream:
|
||||
extractors:
|
||||
- name: "service_log_base"
|
||||
regex: '^(?P<log_level>\S+)\s+(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6})\s+(?P<message>.*)'
|
||||
fields:
|
||||
log_level: "string"
|
||||
timestamp: "time:2006-01-02 15:04:05.000000"
|
||||
message: "string"
|
||||
|
||||
- name: "transfer_id_extraction"
|
||||
regex: '^(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12})\s+(?P<message>.*)'
|
||||
fields:
|
||||
transfer_id: "string"
|
||||
message: "string"
|
||||
|
||||
- name: "transfer_start_in"
|
||||
regex: 'in: Transfer start (?P<thread_info>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<file_count>\d+) size=(?P<size_mb>[0-9.]+) MByte chunksize=(?P<chunk_size>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_rate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) dest=(?P<destination>\S+) sender-id=(?P<sender_id>\S+)'
|
||||
fields:
|
||||
thread_info: "string" # z.B. "1/4" - Typisierung hier schwierig, also String
|
||||
buffers: "int"
|
||||
file_count: "int"
|
||||
size_mb: "float"
|
||||
chunk_size: "int"
|
||||
streams: "int"
|
||||
target_rate: "float"
|
||||
protocol: "string"
|
||||
destination: "string"
|
||||
sender_id: "string"
|
||||
direction: "string" # Wir können statische Felder im Parser injecten oder hier als "implizit" betrachten
|
||||
|
||||
- name: "transfer_start_remote_out"
|
||||
regex: 'out: Start remote transfer to (?P<target>[^\s]+) request executed, duration=(?P<duration>[0-9.]+) s'
|
||||
fields:
|
||||
target: "string"
|
||||
duration: "float"
|
||||
|
||||
- name: "transfer_start_out"
|
||||
regex: 'out: Transfer start (?P<thread_info>\d+/\d+) buffers=(?P<buffers>\d+) files=(?P<file_count>\d+) size=(?P<size_mb>[0-9.]+) MByte chunksize=(?P<chunk_size>\d+) streams=(?P<streams>\d+) target-datarate=(?P<target_rate>[0-9.]+) MByte/s protocol=(?P<protocol>\w+) src=(?P<source>\S+) receiver=(?P<receiver>\S+)'
|
||||
fields:
|
||||
thread_info: "string"
|
||||
buffers: "int"
|
||||
file_count: "int"
|
||||
size_mb: "float"
|
||||
chunk_size: "int"
|
||||
streams: "int"
|
||||
target_rate: "float"
|
||||
protocol: "string"
|
||||
source: "string"
|
||||
receiver: "string"
|
||||
|
||||
- name: "transfer_start_generic"
|
||||
regex: 'out: Start transfer (?P<thread_info>\d+/\d+), src=(?P<source>[^ ]*) dest=(?P<destination>[^ ]*) item\[0\]=(?P<item0>[^ ]*) count=(?P<count>\d+)'
|
||||
fields:
|
||||
thread_info: "string"
|
||||
source: "string"
|
||||
destination: "string"
|
||||
item0: "string"
|
||||
count: "int"
|
||||
|
||||
# ===========================================================================
|
||||
# Transfer Job Manager (TJM)
|
||||
# Deckt ab: tjmServicePattern, tjmTransferNamePattern, tjmTransferIDPattern1/2
|
||||
# ===========================================================================
|
||||
transfer-job-manager:
|
||||
extractors:
|
||||
- name: "service_log_base"
|
||||
regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\s+(?P<log_level>\S+)\s+(?P<pid>\d+).*?\[(?P<correlation_id>[^\]]*)\]\s+\[(?P<username>[^\]]*)\]\s+\[(?P<thread_id>[^\]]*)\]\s+(?P<java_class>.*?)\s+:\s+(?P<message>.*)'
|
||||
fields:
|
||||
timestamp: "time:2006-01-02 15:04:05.000"
|
||||
log_level: "string"
|
||||
pid: "int"
|
||||
correlation_id: "string"
|
||||
username: "string"
|
||||
thread_id: "string"
|
||||
java_class: "string"
|
||||
message: "string"
|
||||
|
||||
- name: "transfer_name_info"
|
||||
regex: '^(?P<transfer_name_raw>\d{8}T\d{6}-[A-Za-z0-9]+-.+?-(?:in|out)) ?: (?P<message>.*)$'
|
||||
fields:
|
||||
transfer_name_raw: "string"
|
||||
message: "string"
|
||||
|
||||
- name: "transfer_id_mid"
|
||||
regex: '(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)'
|
||||
fields:
|
||||
transfer_id: "string"
|
||||
message: "string"
|
||||
|
||||
- name: "transfer_id_prefixed"
|
||||
regex: '(?P<prefix>.*)(?P<transfer_id>\w{8}-\w{4}-\w{4}-\w{4}-\w{12}).*?(?P<message>.*)'
|
||||
fields:
|
||||
prefix: "string"
|
||||
transfer_id: "string"
|
||||
message: "string"
|
||||
|
||||
# ===========================================================================
|
||||
# Access Manager & TCC
|
||||
# Deckt ab: amServicePattern, tccServicePattern
|
||||
# ===========================================================================
|
||||
access-manager:
|
||||
extractors:
|
||||
- name: "spring_boot_log"
|
||||
regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P<log_level>\w+)\s+(?P<pid>\d+)\s+---\s+\[\s*(?P<thread_id>[^\]]*)\]\s+(?P<logger>[\w\.]+)\s*:\s+(?P<message>.*)$'
|
||||
fields:
|
||||
timestamp: "time:2006-01-02T15:04:05.000000Z"
|
||||
log_level: "string"
|
||||
pid: "int"
|
||||
thread_id: "string"
|
||||
logger: "string"
|
||||
message: "string"
|
||||
|
||||
tixel-control-center:
|
||||
extractors:
|
||||
- name: "spring_boot_log"
|
||||
regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+(?P<log_level>\w+)\s+(?P<pid>\d+)\s+---\s+\[\s*(?P<thread_id>[^\]]*)\]\s+(?P<logger>[\w\.]+)\s*:\s+(?P<message>.*)$'
|
||||
fields:
|
||||
timestamp: "time:2006-01-02T15:04:05.000000Z"
|
||||
log_level: "string"
|
||||
pid: "int"
|
||||
thread_id: "string"
|
||||
logger: "string"
|
||||
message: "string"
|
||||
|
||||
# ===========================================================================
|
||||
# Nginx
|
||||
# Deckt ab: nginxAccessPattern
|
||||
# ===========================================================================
|
||||
nginx:
|
||||
extractors:
|
||||
- name: "access_log"
|
||||
regex: '^(?P<client_ip>\S+)\s+\S+\s+(?P<remote_user>\S+)\s+\[(?P<timestamp_nginx>[^\]]+)\]\s+"(?P<request>[^"]+)"\s+(?P<status_code>\d+)\s+(?P<bytes_sent>\d+|-)\s*(?:"(?P<referer>[^"]*)"\s+"(?P<user_agent>[^"]*)")?'
|
||||
fields:
|
||||
client_ip: "string"
|
||||
remote_user: "string"
|
||||
timestamp_nginx: "string"
|
||||
request: "string"
|
||||
status_code: "int"
|
||||
bytes_sent: "int"
|
||||
referer: "string"
|
||||
user_agent: "string"
|
||||
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
#!/bin/bash
|
||||
|
||||
OUTPUT_FILE="/var/log/thesis_training_metrics.csv"
|
||||
IFACE="ens4"
|
||||
PING_TARGET="10.10.2.10"
|
||||
|
||||
echo "timestamp,cpu_user,cpu_sys,cpu_wait,ram_used_mb,disk_read_iops,disk_write_iops,net_rx_kb_s,net_tx_kb_s,rtt_ms" >"$OUTPUT_FILE"
|
||||
|
||||
get_disk_iops() {
|
||||
awk '/(sd[a-z]|vd[a-z] |nvme[0-9]n[0-9])$/ {r+=$4; w+=$8} END {print r+0,w+0}' /proc/diskstats
|
||||
}
|
||||
|
||||
get_net_bytes() {
|
||||
cat "/sys/class/net/$IFACE/statistics/$1"
|
||||
}
|
||||
|
||||
OLD_RX=$(get_net_bytes rx_bytes)
|
||||
OLD_TX=$(get_net_bytes tx_bytes)
|
||||
read OLD_DR OLD_DW <<<$(get_disk_iops)
|
||||
OLD_TS=$(date +%s%N)
|
||||
|
||||
while true; do
|
||||
VMSTAT=$(vmstat 1 2 | tail -1)
|
||||
CPU_US=$(echo "$VMSTAT" | awk '{print $13}')
|
||||
CPU_SY=$(echo "$VMSTAT" | awk '{print $14}')
|
||||
CPU_WA=$(echo "$VMSTAT" | awk '{print $16}')
|
||||
|
||||
RAM_USED=$(free -m | awk '/Mem:/ {print $3}')
|
||||
|
||||
RTT=$(ping -c 1 -W 1 -q "$PING_TARGET" | awk -F'/' '/rtt/ {printf "%.2f", $4}')
|
||||
[ -z "$RTT" ] && RTT="0.0"
|
||||
|
||||
NEW_TS=$(date +%s%N)
|
||||
NEW_RX=$(get_net_bytes rx_bytes)
|
||||
NEW_TX=$(get_net_bytes tx_bytes)
|
||||
read NEW_DR NEW_DW <<<$(get_disk_iops)
|
||||
|
||||
DT=$(echo "scale=3; ($NEW_TS - $OLD_TS) / 1000000000" | bc)
|
||||
|
||||
RX_RATE=$(echo "scale=2; ($NEW_RX - $OLD_RX) / 1024 / $DT" | bc)
|
||||
TX_RATE=$(echo "scale=2; ($NEW_TX - $OLD_TX) / 1024 / $DT" | bc)
|
||||
|
||||
READ_IOPS=$(echo "scale=2; ($NEW_DR - $OLD_DR) / $DT" | bc)
|
||||
WRITE_IOPS=$(echo "scale=2; ($NEW_DW - $OLD_DW) / $DT" | bc)
|
||||
|
||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
|
||||
echo "$TIMESTAMP,$CPU_US,$CPU_SY,$CPU_WA,$RAM_USED,$READ_IOPS,$WRITE_IOPS,$RX_RATE,$TX_RATE,$RTT" >>"$OUTPUT_FILE"
|
||||
|
||||
OLD_RX=$NEW_RX
|
||||
OLD_TX=$NEW_TX
|
||||
OLD_DR=$NEW_DR
|
||||
OLD_DW=$NEW_DW
|
||||
OLD_TS=$NEW_TS
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# #!/bin/bash
|
||||
#
|
||||
# OUTPUT_FILE="/var/log/thesis_resource_baseline.csv"
|
||||
#
|
||||
# if [ ! -f "$OUTPUT_FILE" ]; then
|
||||
# echo "timestamp,cpu_percent,ram_kb,command" > "$OUTPUT_FILE"
|
||||
# fi
|
||||
#
|
||||
# while true; do
|
||||
# PID=$(pgrep -x "watch-tool")
|
||||
#
|
||||
# if [ ! -z "$PID" ]; then
|
||||
# TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
#
|
||||
# STATS=$(pidstat -p $PID -u -r 1 1 | tail -1)
|
||||
#
|
||||
# eval $(ps -p $PID -o %cpu,rss --no-headers | awk '{print "CPU="$1; print "RSS_KB="$2}')
|
||||
#
|
||||
# echo "$TIMESTAMP,$CPU,$RSS_KB,watch-tool" >> "$OUTPUT_FILE"
|
||||
# fi
|
||||
#
|
||||
# sleep 10
|
||||
# done
|
||||
|
||||
OUTPUT_FILE="/var/log/thesis_resource_usage.csv"
|
||||
HEADER="timestamp,system_cpu_usage,system_ram_used_mb,tixstream_cpu,tixstream_rss_mb,watchtool_cpu,watchtool_rss_mb"
|
||||
|
||||
if [ ! -f "$OUTPUT_FILE" ]; then
|
||||
echo "$HEADER" >"$OUTPUT_FILE"
|
||||
fi
|
||||
|
||||
get_process_stats() {
|
||||
local pattern=$1
|
||||
local pid=$(pgrep -f "$pattern" | head -1)
|
||||
|
||||
if [ ! -z "$pid" ]; then
|
||||
ps -p "$pid" -o %cpu,rss --no-headers | awk '{printf "%.2f,%.2f", $1, $2/1024}'
|
||||
else
|
||||
echo "0.00,0.00"
|
||||
fi
|
||||
}
|
||||
|
||||
get_system_stats() {
|
||||
local cpu_sys=$(top -bn1 | grep "Cpu(s)" | awk '{print $2 + $4}')
|
||||
|
||||
local ram_sys=$(free -m | awk '/Mem:/ {print $3}')
|
||||
|
||||
echo "$cpu_sys,$ram_sys"
|
||||
}
|
||||
|
||||
while true; do
|
||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
|
||||
SYS_STATS=$(get_system_stats)
|
||||
|
||||
TIX_STATS=$(get_process_stats "transfer-job-manager")
|
||||
|
||||
TOOL_STATS=$(get_process_stats "watch-tool")
|
||||
|
||||
echo "$TIMESTAMP,$SYS_STATS,$TIX_STATS,$TOOL_STATS" >>"$OUTPUT_FILE"
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
#!/bin/bash
|
||||
|
||||
W1="10.10.2.10" # Worker 1 (Big Files)
|
||||
W2="10.10.3.10" # Worker 2 (Small Files)
|
||||
MY_IP="10.10.1.10" # Worker 0 (Myself)
|
||||
|
||||
# TIXstream Config
|
||||
TIX="/opt/tixel/tixstream/bin/tix"
|
||||
# Quelle Testdaten (auf W0)
|
||||
SRC_LARGE="/data/testfiles/large/5G.dat"
|
||||
SRC_SMALL="/data/testfiles/small/"
|
||||
|
||||
SSH_OPTS="-o StrictHostKeyChecking=no -i /home/baUser/.ssh/id_rsa"
|
||||
|
||||
SCENARIO_LOG="/var/log/thesis_scenario_ground_truth.csv"
|
||||
echo "timestamp,event_type,description" >> "$SCENARIO_LOG"
|
||||
|
||||
log_event() {
|
||||
echo "$(date +"%Y-%m-%d %H:%M:%S"),$1,$2" >> "$SCENARIO_LOG"
|
||||
}
|
||||
|
||||
echo "Starting Thesis-Baseline-Scenario..."
|
||||
|
||||
while true; do
|
||||
log_event "NORMAL_LOAD" "Upload Large File to W1"
|
||||
$TIX submit -s "$SRC_LARGE" -t "tixel://$W1/incoming/" --wait
|
||||
|
||||
sleep 10
|
||||
|
||||
log_event "NORMAL_LOAD" "Upload Small Files to W2"
|
||||
$TIX submit -s "$SRC_SMALL" -t "tixel://$W2/incoming/" -r --wait
|
||||
|
||||
sleep 10
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# 3. Interferenz: Eingehender Traffic (W1 -> W0)
|
||||
# Wir triggern das REMOTE auf W1 via SSH!
|
||||
# ---------------------------------------------------------
|
||||
log_event "INTERFERENCE" "Incoming Transfer from W1"
|
||||
# W1 sendet eine generierte Datei an mich zurück
|
||||
# (Voraussetzung: Auf W1 existiert /data/testfiles/large/1G.dat)
|
||||
ssh $SSH_OPTS baUser@$W1 "$TIX submit -s /data/testfiles/large/1G.dat -t tixel://$MY_IP/incoming/ --wait"
|
||||
|
||||
# Aufräumen bei mir (W0 ist KEIN Blackhole, wir löschen kontrolliert)
|
||||
rm -rf /data/tixel/incoming/*
|
||||
|
||||
sleep 10
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# 4. Stress-Test: Bidirektional (W0->W1 UND W2->W0)
|
||||
# Simuliert "Busy Day" (Normal, aber hohe Last)
|
||||
# ---------------------------------------------------------
|
||||
log_event "HIGH_LOAD" "Bidirectional Transfer"
|
||||
# Start Upload Background
|
||||
$TIX submit -s "$SRC_LARGE" -t "tixel://$W1/incoming/" &
|
||||
PID_UP=$!
|
||||
|
||||
# Trigger Download Background
|
||||
ssh $SSH_OPTS baUser@$W2 "$TIX submit -s /data/testfiles/small/ -t tixel://$MY_IP/incoming/ -r" &
|
||||
|
||||
# Warten bis Upload fertig
|
||||
wait $PID_UP
|
||||
|
||||
# Aufräumen
|
||||
rm -rf /data/tixel/incoming/*
|
||||
|
||||
sleep 30
|
||||
done
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
#!/bin/bash
|
||||
|
||||
W0="thesis-worker-0"
|
||||
W1="thesis-worker-1"
|
||||
W2="thesis-worker-2"
|
||||
PORT="60000"
|
||||
AUTH="YWRtaW46dmVyeXNlY3JldA=="
|
||||
|
||||
LOG="/var/log/thesis_scenario_ground_truth.csv"
|
||||
[ ! -f "$LOG" ] && echo "timestamp,action,source,target" >"$LOG"
|
||||
|
||||
log_gt() { echo "$(date +"%Y-%m-%d %H:%M:%S"),$1,$2,$3" >>"$LOG"; }
|
||||
|
||||
wait_for_api() {
|
||||
local host=$1
|
||||
echo "Prüfe Verfügbarkeit von TIXstream auf $host..."
|
||||
|
||||
while true; do
|
||||
HTTP_CODE=$(curl --write-out "%{http_code}" --silent --output /dev/null \
|
||||
--url "http://$host:$PORT/transfer-job-manager/v1/jobs?limit=1" \
|
||||
--header "Authorization: Basic $AUTH")
|
||||
|
||||
if [ "$HTTP_CODE" -eq 200 ]; then
|
||||
echo "API auf $host ist ONLINE (Code 200)."
|
||||
break
|
||||
else
|
||||
echo "Warte auf API ($host)... Status: $HTTP_CODE. Retry in 10s."
|
||||
sleep 10
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
trigger_job() {
|
||||
local host=$1
|
||||
local dest_sys=$2
|
||||
local file_uri=$3
|
||||
local desc=$4
|
||||
|
||||
DATA=$(jq -n \
|
||||
--arg desc "$desc" \
|
||||
--arg dest_sys "$dest_sys" \
|
||||
--arg file "$file_uri" \
|
||||
'{
|
||||
"description": $desc,
|
||||
"schedule": "NOW",
|
||||
"destination_system": $dest_sys,
|
||||
"destination_share": "local_sync_destination",
|
||||
"source_file_uris": [$file],
|
||||
"wait_for_publish": false,
|
||||
"flat_file_mode_disabled": false
|
||||
}')
|
||||
|
||||
curl --request POST \
|
||||
--url "http://$host:$PORT/transfer-job-manager/v1/jobs" \
|
||||
--header "Authorization: Basic $AUTH" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "$DATA" --silent >/dev/null
|
||||
}
|
||||
|
||||
wait_for_api "$W0"
|
||||
|
||||
wait_for_api "$W1"
|
||||
wait_for_api "$W2"
|
||||
|
||||
echo "Alle Systeme bereit. Starte Baseline-Szenario Loop..."
|
||||
log_gt "SYSTEM_READY" "ALL" "Orchestrator started"
|
||||
INCOMING_DIR="/local_testdata/test-destination"
|
||||
|
||||
while true; do
|
||||
log_gt "START_LARGE" "W0" "W1"
|
||||
trigger_job "$W0" "Worker Node 1" "local_sync_source/5g.mxf" "Thesis Large File"
|
||||
sleep 60
|
||||
|
||||
log_gt "START_SMALL" "W0" "W2"
|
||||
for i in {1..10}; do
|
||||
trigger_job "$W0" "Worker Node 2" "local_sync_source/small_$i.dat" "Thesis Small File $i"
|
||||
done
|
||||
sleep 30
|
||||
|
||||
log_gt "START_INTERFERENCE" "W1" "W0"
|
||||
trigger_job "$W1" "Worker Node 0" "local_sync_source/5g.mxf" "Interference Load"
|
||||
|
||||
sleep 60
|
||||
rm -rf "${INCOMING_DIR:?}/"*
|
||||
done
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[Unit]
|
||||
Description=Thesis Watch Tool - Metrics Collector
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=root
|
||||
WorkingDirectory=/opt/watch-tool
|
||||
ExecStart=/opt/watch-tool/watch-tool
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
network_scenarios:
|
||||
# ---------------------------------------------------------------------------
|
||||
# 00. Baseline
|
||||
# ---------------------------------------------------------------------------
|
||||
baseline:
|
||||
description: "Normalzustand"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 01. Slow Connection
|
||||
# ---------------------------------------------------------------------------
|
||||
slow-connection:
|
||||
description: "Langsam & Latenz (Satellit)"
|
||||
interfaces:
|
||||
ens4:
|
||||
- {
|
||||
root: true,
|
||||
type: "tbf",
|
||||
args: "rate 256kbit burst 1540 latency 50ms",
|
||||
}
|
||||
ens5:
|
||||
- {
|
||||
root: true,
|
||||
type: "tbf",
|
||||
args: "rate 256kbit burst 1540 latency 50ms",
|
||||
}
|
||||
ens6:
|
||||
- { root: true, handle: "1:", type: "netem", args: "delay 300ms" }
|
||||
- {
|
||||
parent: "1:1",
|
||||
handle: "10:",
|
||||
type: "tbf",
|
||||
args: "rate 512kbit burst 1540 latency 50ms",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 02. High Latency
|
||||
# ---------------------------------------------------------------------------
|
||||
high-latency:
|
||||
description: "Hohe Latenz"
|
||||
interfaces:
|
||||
ens4:
|
||||
- {
|
||||
root: true,
|
||||
type: "netem",
|
||||
args: "delay 200ms 20ms distribution normal",
|
||||
}
|
||||
ens5:
|
||||
- {
|
||||
root: true,
|
||||
type: "netem",
|
||||
args: "delay 200ms 20ms distribution normal",
|
||||
}
|
||||
ens6:
|
||||
- {
|
||||
root: true,
|
||||
type: "netem",
|
||||
args: "delay 350ms 30ms distribution normal",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 03. Packet Loss
|
||||
# ---------------------------------------------------------------------------
|
||||
packet-loss:
|
||||
description: "Paketverlust"
|
||||
interfaces:
|
||||
ens4: [{ root: true, type: "netem", args: "loss 15% 10%" }]
|
||||
ens5: [{ root: true, type: "netem", args: "loss 15% 10%" }]
|
||||
ens6: [{ root: true, type: "netem", args: "loss 5% 2%" }]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 06. Congestion
|
||||
# ---------------------------------------------------------------------------
|
||||
congestion:
|
||||
description: "Überlastung (Delay + Loss + Rate Limit)"
|
||||
interfaces:
|
||||
ens4:
|
||||
- {
|
||||
root: true,
|
||||
handle: "1:",
|
||||
type: "netem",
|
||||
args: "delay 50ms 20ms loss 3% 5%",
|
||||
}
|
||||
- {
|
||||
parent: "1:1",
|
||||
handle: "10:",
|
||||
type: "tbf",
|
||||
args: "rate 2mbit burst 32kbit latency 800ms",
|
||||
}
|
||||
ens5:
|
||||
- {
|
||||
root: true,
|
||||
handle: "1:",
|
||||
type: "netem",
|
||||
args: "delay 50ms 20ms loss 3% 5%",
|
||||
}
|
||||
- {
|
||||
parent: "1:1",
|
||||
handle: "10:",
|
||||
type: "tbf",
|
||||
args: "rate 2mbit burst 32kbit latency 800ms",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 07. Partial Outage (IPTables Block)
|
||||
# ---------------------------------------------------------------------------
|
||||
partial-outage:
|
||||
description: "Verbindung W0 <-> W1 tot"
|
||||
blocks:
|
||||
- { src: "ens4", dst: "ens5" }
|
||||
- { src: "ens5", dst: "ens4" }
|
||||
interfaces:
|
||||
ens6: [{ root: true, type: "netem", args: "delay 10ms" }]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 08. Flapping
|
||||
# ---------------------------------------------------------------------------
|
||||
flapping:
|
||||
description: "Wackelkontakt (30s an/aus)"
|
||||
flapping_enabled: true
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
---
|
||||
- name: "Szenario-Info: {{ network_scenarios[scenario].description }}"
|
||||
debug:
|
||||
msg: "Activating Szenario '{{ scenario }}'"
|
||||
|
||||
# --- 1. CLEANUP
|
||||
|
||||
- name: Stop Flapping Service (falls aktiv)
|
||||
systemd:
|
||||
name: flapping_simulation
|
||||
state: stopped
|
||||
enabled: no
|
||||
ignore_errors: true
|
||||
|
||||
- name: Reset Traffic Control
|
||||
shell: "tc qdisc del dev {{ item }} root"
|
||||
loop: [ens4, ens5, ens6]
|
||||
ignore_errors: true
|
||||
changed_when: false
|
||||
|
||||
- name: Reset IPTables Blocks
|
||||
iptables:
|
||||
chain: FORWARD
|
||||
action: flush
|
||||
changed_when: false
|
||||
|
||||
# --- 2. INSTALL FLAPPING
|
||||
|
||||
- name: Installing Flapping script & service
|
||||
block:
|
||||
- template:
|
||||
src: flapping_service.sh.j2
|
||||
dest: /usr/local/bin/flapping_simulation.sh
|
||||
mode: "0755"
|
||||
- template:
|
||||
src: flapping.service.j2
|
||||
dest: /etc/systemd/system/flapping_simulation.service
|
||||
- systemd:
|
||||
name: flapping_simulation
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
when: network_scenarios[scenario].flapping_enabled | default(false)
|
||||
|
||||
# --- 3. APPLY TRAFFIC CONTROL
|
||||
|
||||
- name: Apply Complex TC Rules
|
||||
shell: >
|
||||
tc qdisc add dev {{ item.0.key }}
|
||||
{% if item.1.root | default(false) %}root{% else %}parent {{ item.1.parent }}{% endif %}
|
||||
{% if item.1.handle is defined %}handle {{ item.1.handle }}{% endif %}
|
||||
{{ item.1.type }}
|
||||
{{ item.1.args }}
|
||||
loop: "{{ network_scenarios[scenario].interfaces | default({}) | dict2items | subelements('value') }}"
|
||||
when: network_scenarios[scenario].interfaces is defined
|
||||
|
||||
# --- 4. APPLY IPTABLES BLOCKS ---
|
||||
|
||||
- name: Apply Static Blocks
|
||||
iptables:
|
||||
chain: FORWARD
|
||||
in_interface: "{{ item.src }}"
|
||||
out_interface: "{{ item.dst }}"
|
||||
jump: DROP
|
||||
loop: "{{ network_scenarios[scenario].blocks | default([]) }}"
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Network Flapping Simulation
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/local/bin/flapping_simulation.sh
|
||||
ExecStop=/usr/bin/iptables -D FORWARD -i ens4 -o ens5 -j DROP ; /usr/bin/iptables -D FORWARD -i ens5 -o ens4 -j DROP
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
while true; do
|
||||
iptables -I FORWARD 1 -i ens4 -o ens5 -j DROP
|
||||
iptables -I FORWARD 1 -i ens5 -o ens4 -j DROP
|
||||
sleep 30
|
||||
|
||||
iptables -D FORWARD -i ens4 -o ens5 -j DROP 2>/dev/null || true
|
||||
iptables -D FORWARD -i ens5 -o ens4 -j DROP 2>/dev/null || true
|
||||
sleep 30
|
||||
done
|
||||
57
infrastructure/ansible/roles/router/tasks/main.yml
Normal file
57
infrastructure/ansible/roles/router/tasks/main.yml
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
---
|
||||
- name: Aktiviere IPv4 Forwarding (Kernel)
|
||||
sysctl:
|
||||
name: net.ipv4.ip_forward
|
||||
value: "1"
|
||||
sysctl_set: yes
|
||||
state: present
|
||||
reload: yes
|
||||
|
||||
- name: Installiere iptables-persistent
|
||||
apt:
|
||||
name: iptables-persistent
|
||||
state: present
|
||||
|
||||
- name: Spüle existierende IPTables Regeln
|
||||
iptables:
|
||||
chain: "{{ item }}"
|
||||
flush: yes
|
||||
loop:
|
||||
- INPUT
|
||||
- FORWARD
|
||||
- OUTPUT
|
||||
|
||||
- name: Ermittle WAN Interface
|
||||
shell: ip route show default | awk '/default/ {print $5}'
|
||||
register: wan_interface
|
||||
changed_when: false
|
||||
|
||||
- name: Aktiviere Masquerading (NAT) auf dem WAN Interface
|
||||
iptables:
|
||||
table: nat
|
||||
chain: POSTROUTING
|
||||
out_interface: "{{ wan_interface.stdout }}"
|
||||
jump: MASQUERADE
|
||||
|
||||
- name: Erlaube Forwarding für internes Netz (10.10.0.0/16)
|
||||
iptables:
|
||||
chain: FORWARD
|
||||
source: 10.10.0.0/16
|
||||
destination: 10.10.0.0/16
|
||||
jump: ACCEPT
|
||||
|
||||
- name: Erlaube Forwarding von Intern ins Internet (Established)
|
||||
iptables:
|
||||
chain: FORWARD
|
||||
ctstate: ESTABLISHED,RELATED
|
||||
jump: ACCEPT
|
||||
|
||||
- name: Erlaube Forwarding von Intern ins Internet (New)
|
||||
iptables:
|
||||
chain: FORWARD
|
||||
source: 10.10.0.0/16
|
||||
out_interface: "{{ wan_interface.stdout }}"
|
||||
jump: ACCEPT
|
||||
|
||||
- name: Speichere IPTables Regeln dauerhaft
|
||||
shell: netfilter-persistent save
|
||||
13
infrastructure/ansible/roles/smtplog/handlers/main.yml
Normal file
13
infrastructure/ansible/roles/smtplog/handlers/main.yml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
|
||||
- name: Restart smtplog
|
||||
service:
|
||||
name: smtplog
|
||||
state: restarted
|
||||
|
||||
- name: Restart smtpweb
|
||||
service:
|
||||
name: smtpweb
|
||||
state: restarted
|
||||
|
||||
# vim:ft=ansible
|
||||
175
infrastructure/ansible/roles/smtplog/tasks/main.yml
Normal file
175
infrastructure/ansible/roles/smtplog/tasks/main.yml
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
---
|
||||
|
||||
- name: Set current_host_config variable to access parameters referencing this host
|
||||
set_fact:
|
||||
current_host_config: "{{ item.value }}"
|
||||
when: item.value.hostname == ansible_fqdn
|
||||
with_dict: "{{ configs.host_config }}"
|
||||
|
||||
- name: install python-setuptools for Python interpreter
|
||||
package:
|
||||
name: python-setuptools
|
||||
state: present
|
||||
become: true
|
||||
when: ansible_distribution_major_version == "7"
|
||||
|
||||
- name: Deinstall postfix
|
||||
package:
|
||||
name: postfix
|
||||
state: absent
|
||||
become: true
|
||||
ignore_errors: true
|
||||
|
||||
- name: Install python 3.6
|
||||
package:
|
||||
name: python36
|
||||
state: present
|
||||
become: true
|
||||
when: ansible_distribution == "CentOS"
|
||||
|
||||
#Pip module has a parameter for system packages but it was added in 2.17 which might
|
||||
# be to new for some developer work stations
|
||||
#
|
||||
- name: Install aiosmtpd for pip3
|
||||
command: pip3 install --break-system-packages aiosmtpd
|
||||
become: true
|
||||
when: ansible_distribution == "Ubuntu" and ansible_distribution_major_version == "24"
|
||||
|
||||
- name: Install aiosmtpd for pip3
|
||||
command: pip3 install aiosmtpd
|
||||
become: true
|
||||
when: (ansible_distribution == "Ubuntu" and (ansible_distribution_major_version == "20" or ansible_distribution_major_version == "22"))
|
||||
|
||||
- name: Install aiosmtpd for pip3
|
||||
command: pip3 install aiosmtpd
|
||||
become: true
|
||||
when: ((ansible_os_family == "RedHat" and (ansible_distribution_major_version == "7" or ansible_distribution_major_version == "8")) or ansible_distribution == "Rocky" )
|
||||
|
||||
- name: Create smtplog directory
|
||||
file:
|
||||
path: "{{ remote_deployment_dir }}/smtplog"
|
||||
state: directory
|
||||
|
||||
- name: Find smtpservices package
|
||||
local_action:
|
||||
module: find
|
||||
paths: "{{ configs.deployment_dir }}"
|
||||
patterns: "smtplog*.zip"
|
||||
file_type: file
|
||||
register: found_packages
|
||||
become: false
|
||||
|
||||
- name: Order packages to install
|
||||
set_fact:
|
||||
latest_package: "{{ found_packages.files | sort(attribute='mtime',reverse=true) | first }}"
|
||||
|
||||
- name: Unpack smtpservice
|
||||
unarchive:
|
||||
src: "{{ latest_package.path }}"
|
||||
dest: "{{ remote_deployment_dir }}/smtplog"
|
||||
|
||||
- name: Set the smtplog/smtpweb script location
|
||||
find:
|
||||
path: "{{ remote_deployment_dir }}/smtplog"
|
||||
pattern: "smtplog*"
|
||||
file_type: directory
|
||||
register: smtplog_deploy_dir
|
||||
|
||||
- name: Set smtplog/smtpweb path
|
||||
set_fact:
|
||||
smtplog_path: "{{ smtplog_deploy_dir.files | map(attribute='path') | join('') }}"
|
||||
|
||||
- name: Install smtplog script
|
||||
copy:
|
||||
src: "{{ smtplog_path }}/smtplog.py"
|
||||
dest: /usr/local/bin/smtplog.py
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
remote_src: true
|
||||
become: true
|
||||
|
||||
- name: Install smtplog systemd unit
|
||||
copy:
|
||||
src: "{{ smtplog_path }}/smtplog.service"
|
||||
dest: "{{ smtplog_service_file }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0644
|
||||
remote_src: true
|
||||
become: true
|
||||
|
||||
- name: Set smtplog log directory
|
||||
lineinfile:
|
||||
dest: "{{ smtplog_service_file }}"
|
||||
regexp: '^WorkingDirectory.*$'
|
||||
line: "WorkingDirectory={{ smtplog_logdir }}"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Set smtplog service start command line
|
||||
lineinfile:
|
||||
dest: "{{ smtplog_service_file }}"
|
||||
regexp: '^ExecStart.*$'
|
||||
line: "ExecStart={{ smtplog_dir }}/smtplog.py -a {{ current_host_config.smtplog_server_ip | default(current_host_config.ip) }} -d {{smtplog_maildir}} -p {{ smtplog_port }}"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Create log directory
|
||||
file:
|
||||
path: "{{ smtplog_logdir }}"
|
||||
state: directory
|
||||
mode: 0755
|
||||
become: true
|
||||
|
||||
- name: Install smtpweb script
|
||||
copy:
|
||||
src: "{{ smtplog_path }}/smtpweb.py"
|
||||
dest: /usr/local/bin/smtpweb.py
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
remote_src: true
|
||||
become: true
|
||||
|
||||
- name: Install smtpweb systemd unit
|
||||
copy:
|
||||
src: "{{ smtplog_path }}/smtpweb.service"
|
||||
dest: "{{ smtpweb_service_file }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0644
|
||||
remote_src: true
|
||||
become: true
|
||||
|
||||
- name: Set smtpweb log directory
|
||||
lineinfile:
|
||||
dest: "{{ smtpweb_service_file }}"
|
||||
regexp: '^WorkingDirectory.*$'
|
||||
line: "WorkingDirectory={{ smtpweb_logdir }}"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Set smtpweb service start command line
|
||||
lineinfile:
|
||||
dest: "{{ smtpweb_service_file }}"
|
||||
regexp: '^ExecStart.*$'
|
||||
line: "ExecStart={{ smtplog_dir }}/smtpweb.py -a 127.0.0.1 -d {{smtplog_maildir}} -p {{ smtpweb_port }}"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Create smptweb log directory
|
||||
file:
|
||||
path: "{{ smtpweb_logdir }}"
|
||||
state: directory
|
||||
mode: 0755
|
||||
become: true
|
||||
|
||||
- name: Start and enable smtplog
|
||||
service: name={{ item }} state=started enabled=True
|
||||
with_items:
|
||||
- smtplog
|
||||
- smtpweb
|
||||
become: true
|
||||
|
||||
# vim:ft=ansible
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
- name: Restart Proftpd
|
||||
service:
|
||||
name: proftpd
|
||||
state: restarted
|
||||
100
infrastructure/ansible/roles/transfer_node/tasks/main.yml
Normal file
100
infrastructure/ansible/roles/transfer_node/tasks/main.yml
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
---
|
||||
- name: Install MariaDB server and client
|
||||
apt:
|
||||
name:
|
||||
- mariadb-server
|
||||
- python3-mysqldb
|
||||
state: present
|
||||
|
||||
- name: Start and activate MariaDB
|
||||
service:
|
||||
name: mariadb
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
- name: Erstelle TIXstream Datenbanken
|
||||
community.mysql.mysql_db:
|
||||
name: "{{ item }}"
|
||||
state: present
|
||||
login_unix_socket: /var/run/mysqld/mysqld.sock
|
||||
loop:
|
||||
- transfer_job_manager
|
||||
- access_manager
|
||||
become: true
|
||||
|
||||
- name: Erstelle DB User 'tixel' (localhost)
|
||||
community.mysql.mysql_user:
|
||||
name: tixel
|
||||
password: tixel
|
||||
host: "localhost"
|
||||
priv: "*.*:ALL"
|
||||
state: present
|
||||
login_unix_socket: /var/run/mysqld/mysqld.sock
|
||||
become: true
|
||||
|
||||
- name: Erstelle DB User 'tixel' (Any Host)
|
||||
community.mysql.mysql_user:
|
||||
name: tixel
|
||||
password: tixel
|
||||
host: "%"
|
||||
priv: "*.*:ALL"
|
||||
state: present
|
||||
login_unix_socket: /var/run/mysqld/mysqld.sock
|
||||
become: true
|
||||
|
||||
- name: Install Nginx
|
||||
apt:
|
||||
name: nginx
|
||||
state: present
|
||||
|
||||
- name: Start and activate Nginx
|
||||
service:
|
||||
name: nginx
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
- name: Install ProFTPD with MySQL Modul
|
||||
apt:
|
||||
name:
|
||||
- proftpd-core
|
||||
- proftpd-mod-mysql
|
||||
- proftpd-mod-crypto
|
||||
state: present
|
||||
|
||||
- name: Create FTP Group
|
||||
group:
|
||||
name: ftpgroup
|
||||
state: present
|
||||
|
||||
- name: Create FTP User
|
||||
user:
|
||||
name: ftpuser
|
||||
group: ftpgroup
|
||||
shell: /bin/false
|
||||
home: /var/www/upload
|
||||
create_home: yes
|
||||
|
||||
- name: Create Upload-Dir
|
||||
file:
|
||||
path: /var/www/upload
|
||||
state: directory
|
||||
owner: ftpuser
|
||||
group: ftpgroup
|
||||
mode: "0775"
|
||||
|
||||
- name: Konfiguriere ProFTPD
|
||||
template:
|
||||
src: proftpd.conf.j2
|
||||
dest: /etc/proftpd/proftpd.conf
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Restart Proftpd
|
||||
|
||||
- name: Installiere System-Abhängigkeiten für TIXstream (C++ Binaries)
|
||||
apt:
|
||||
name:
|
||||
- libssh2-1-dev
|
||||
- libssh2-1
|
||||
state: present
|
||||
update_cache: yes
|
||||
Loading…
Add table
Add a link
Reference in a new issue