From 24f74e4d29b9261ec8d7c9137522ecf83fa43998 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Wed, 15 Nov 2023 22:44:26 +0100 Subject: [PATCH 01/61] wip --- .gitignore | 3 +- ansible/bambichecker.yml | 1 + ansible/bambiengine.yml | 1 + ansible/bambirouter.yml | 2 +- ansible/bambivulnbox.yml | 2 +- ansible/config_bambi.yml.sample | 2 +- .../bambi-wireguard-router/tasks/main.yml | 5 + ansible/roles/vuln_checkers/meta/main.yml | 1 - ansible/roles/vuln_services/meta/main.yml | 1 - .../roles/wireguard_checker/tasks/main.yml | 5 + ansible/roles/wireguard_engine/tasks/main.yml | 5 + config/gen_config.sh | 57 --- config/internal_router/gen_keys.py | 270 ------------- config/openvpn_configs/gen_configs.sh | 117 ------ config/passwords/gen_passwords.sh | 1 - config/wireguard_router/gen_keys.py | 177 --------- configgen/.gitignore | 0 configgen/README.md | 0 configgen/configgen/__init__.py | 94 +++++ configgen/configgen/gen_wireguard_game.py | 157 ++++++++ configgen/configgen/gen_wireguard_internal.py | 134 +++++++ configgen/configgen/util.py | 144 +++++++ configgen/poetry.lock | 243 ++++++++++++ configgen/pyproject.toml | 37 ++ docker-compose.yml | 9 +- terraform/bambictf.tf | 357 ++++++------------ terraform/terraform.tfvars.sample | 9 +- terraform/user_data_checker.tftpl | 19 + terraform/user_data_engine.tftpl | 21 ++ terraform/user_data_router.tftpl | 32 ++ terraform/user_data_vulnbox.tftpl | 21 ++ 31 files changed, 1040 insertions(+), 887 deletions(-) create mode 100644 ansible/roles/wireguard_checker/tasks/main.yml create mode 100644 ansible/roles/wireguard_engine/tasks/main.yml delete mode 100755 config/gen_config.sh delete mode 100755 config/internal_router/gen_keys.py delete mode 100755 config/openvpn_configs/gen_configs.sh delete mode 100755 config/passwords/gen_passwords.sh delete mode 100755 config/wireguard_router/gen_keys.py create mode 100644 configgen/.gitignore create mode 100644 configgen/README.md create mode 100644 configgen/configgen/__init__.py create mode 100644 configgen/configgen/gen_wireguard_game.py create mode 100644 configgen/configgen/gen_wireguard_internal.py create mode 100644 configgen/configgen/util.py create mode 100644 configgen/poetry.lock create mode 100644 configgen/pyproject.toml create mode 100644 terraform/user_data_checker.tftpl create mode 100644 terraform/user_data_engine.tftpl create mode 100644 terraform/user_data_router.tftpl create mode 100644 terraform/user_data_vulnbox.tftpl diff --git a/.gitignore b/.gitignore index 100aac6..e656412 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ config*.yml .env id_ed25519 .secrets -ctf*json \ No newline at end of file +ctf*json +*.pyc diff --git a/ansible/bambichecker.yml b/ansible/bambichecker.yml index 9d24e6c..183f09c 100644 --- a/ansible/bambichecker.yml +++ b/ansible/bambichecker.yml @@ -9,6 +9,7 @@ - bambi-ssh-keys - firewall - wireguard + - wireguard_checker - role: filebeat vars: elk: 192.168.3.0 diff --git a/ansible/bambiengine.yml b/ansible/bambiengine.yml index e4c1bb4..fcc7a4b 100644 --- a/ansible/bambiengine.yml +++ b/ansible/bambiengine.yml @@ -10,6 +10,7 @@ - firewall - wireguard - enoengine + - wireguard_engine - role: filebeat vars: elk: 192.168.3.0 diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index e460f96..aa20523 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -23,4 +23,4 @@ program_list: - "tmux" - "git" - - teamvpn-configs + #- teamvpn-configs diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index a0afc9c..fed8949 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -25,4 +25,4 @@ vars: pip_list: - pwntools - - bambi-openvpn-team-server + #- bambi-openvpn-team-server diff --git a/ansible/config_bambi.yml.sample b/ansible/config_bambi.yml.sample index 319b482..0b17b77 100644 --- a/ansible/config_bambi.yml.sample +++ b/ansible/config_bambi.yml.sample @@ -2,4 +2,4 @@ vulnerable_services: stonksexchange: https://github.com/ldruschk/bambi-service-stonksexchange.git github_ssh_keys: [ ldruschk, -] \ No newline at end of file +] diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml index 1395326..5c6d519 100644 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ b/ansible/roles/bambi-wireguard-router/tasks/main.yml @@ -1,4 +1,9 @@ --- + - name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/routers/" + dest: /etc/wireguard/ + - name: allow wireguard input traffic (game) iptables: chain: INPUT diff --git a/ansible/roles/vuln_checkers/meta/main.yml b/ansible/roles/vuln_checkers/meta/main.yml index acda303..6533535 100644 --- a/ansible/roles/vuln_checkers/meta/main.yml +++ b/ansible/roles/vuln_checkers/meta/main.yml @@ -1,4 +1,3 @@ dependencies: - role: "docker" -- role: "programs" - role: "bambi-ssh-keys" diff --git a/ansible/roles/vuln_services/meta/main.yml b/ansible/roles/vuln_services/meta/main.yml index acda303..6533535 100644 --- a/ansible/roles/vuln_services/meta/main.yml +++ b/ansible/roles/vuln_services/meta/main.yml @@ -1,4 +1,3 @@ dependencies: - role: "docker" -- role: "programs" - role: "bambi-ssh-keys" diff --git a/ansible/roles/wireguard_checker/tasks/main.yml b/ansible/roles/wireguard_checker/tasks/main.yml new file mode 100644 index 0000000..b14e21e --- /dev/null +++ b/ansible/roles/wireguard_checker/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/checkers/" + dest: /etc/wireguard/ diff --git a/ansible/roles/wireguard_engine/tasks/main.yml b/ansible/roles/wireguard_engine/tasks/main.yml new file mode 100644 index 0000000..83d3aac --- /dev/null +++ b/ansible/roles/wireguard_engine/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/engine.conf" + dest: /etc/wireguard/internal.conf diff --git a/config/gen_config.sh b/config/gen_config.sh deleted file mode 100755 index c72cf3e..0000000 --- a/config/gen_config.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -TEAM_COUNT=$(if [ -z "$TEAM_COUNT" ]; then echo 255; else echo "$TEAM_COUNT"; fi) -GATEWAY_COUNT=$(if [ -z "$GATEWAY_COUNT" ]; then echo 2; else echo "$GATEWAY_COUNT"; fi) -CHECKER_COUNT=$(if [ -z "$CHECKER_COUNT" ]; then echo 4; else echo "$CHECKER_COUNT"; fi) - -mkdir -p "export" -mkdir -p "internal_router" -mkdir -p "wireguard_router" -mkdir -p "openvpn_configs" -mkdir -p "passwords" - -( - cd "internal_router" - ./gen_keys.py $TEAM_COUNT $GATEWAY_COUNT $CHECKER_COUNT -) -( - cd "wireguard_router" - ./gen_keys.py $TEAM_COUNT $GATEWAY_COUNT -) -( - cd "openvpn_configs" - ./gen_configs.sh $TEAM_COUNT $GATEWAY_COUNT -) -( - cd "passwords" - ./gen_passwords.sh $TEAM_COUNT -) - -for i in $(seq 1 ${TEAM_COUNT}); do - mkdir -p "export/team$i" - cat <<OUTEREOF > "export/team$i/user_data.sh" -#!/bin/sh -cat <<EOF >> /etc/wireguard/game.conf -$(cat "wireguard_router/team_configs/team${i}.conf" ) -EOF -systemctl enable wg-quick@game -systemctl start wg-quick@game - -for service in \$(ls /services/); do - cd "/services/\$service" - if [ -f "/services/\$service/setup.sh" ]; then - /services/\$service/setup.sh - fi - docker-compose up -d & -done - -cat <<EOF | passwd -$(cat passwords/team${i}.txt | tr -d '\n') -$(cat passwords/team${i}.txt | tr -d '\n') -EOF -OUTEREOF - - cp "passwords/team${i}.txt" "export/team${i}/root.pw" - cp "openvpn_configs/team${i}/client.conf" "export/team${i}/client.conf" - cp "wireguard_router/team_configs/team${i}.conf" "export/team${i}/wireguard.conf" -done diff --git a/config/internal_router/gen_keys.py b/config/internal_router/gen_keys.py deleted file mode 100755 index dfb809b..0000000 --- a/config/internal_router/gen_keys.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 - -import os -import subprocess -import sys -from dataclasses import dataclass -from typing import List, Optional - - -GAME_NETWORK = "10.0.0.0/8" -TEAM_IP_PREFIX_TEMPLATE = "10.%d.%d." -TEAM_IP_SUBNET_SIZE = "/24" -INTERNAL_NETWORK = "192.168.0.0/20" -ROUTER_IP_TEMPLATE = "192.168.0.%d/32" -CHECKER_IP_TEMPLATE = "192.168.1.%d/32" -ENGINE_IP_ADDRESS = "192.168.1.0/32" -ELK_IP_ADDRESS = "192.168.3.0/32" - -GATEWAY_DOMAIN_TEMPLATE = "vpn%d.bambi.ovh" -GATEWAY_LISTEN_PORT = 51821 - -ENGINE_DOMAIN = "engine-vpn.bambi.ovh" -ENGINE_LISTEN_PORT = 51821 - -ELK_DOMAIN = "elk-vpn.bambi.ovh" -ELK_LISTEN_PORT = 51821 - - -@dataclass -class Peer: - public_key: str - allowed_ips: List[str] - endpoint: Optional[str] - - -@dataclass -class WireguardConfig: - private_key: str - public_key: str - address: str - peers: List[Peer] - listen_port: Optional[int] - - -@dataclass -class GatewayConfig(WireguardConfig): - allowed_ips: List[str] - domain_name: str - - -@dataclass -class CheckerConfig(WireguardConfig): - pass - - -def gen_wireguard_keys(): - privkey_raw = subprocess.check_output(["wg", "genkey"]) - privkey = privkey_raw.strip().decode() - result = subprocess.run(["wg", "pubkey"], stdout=subprocess.PIPE, input=privkey_raw) - pubkey = result.stdout.strip().decode() - return privkey, pubkey - - -def create_interface_section(config: WireguardConfig): - raw = f"""[Interface] -Address = {config.address} -PrivateKey = {config.private_key} -""" - if config.listen_port is not None: - raw += f"ListenPort = {config.listen_port}\n" - return raw - - -def create_peer_section(peer: Peer): - allowed_ips = ", ".join(peer.allowed_ips) - raw = f"""[Peer] -PublicKey = {peer.public_key} -AllowedIPs = {allowed_ips} -""" - if peer.endpoint is not None: - raw += f"""PersistentKeepalive = 15 -Endpoint = {peer.endpoint} -""" - return raw - - -def create_config_file(config: WireguardConfig): - sections = [create_interface_section(config)] - for peer in config.peers: - sections.append(create_peer_section(peer)) - return "\n".join(sections) - - -if len(sys.argv) < 4: - print(f"Please specify a number of teams, gateways and checkers. Usage: {sys.argv[0]} <no of teams> <no of gateways> <no of checkers>") - sys.exit(1) - -try: - num_teams = int(sys.argv[1]) - if num_teams < 1 or num_teams > 2000: - print("The number of teams must be between 1 and 2000.") - sys.exit(1) -except Exception as ex: - print(f"Could not parse number of teams: {ex}") - sys.exit(1) - -try: - num_gateways = int(sys.argv[2]) - if num_gateways < 1 or num_gateways > num_teams or num_gateways > 250: - print("The number of gateways must be between 1 and min(250, <no of teams>)") - sys.exit(1) -except Exception as ex: - print(f"Could not parse number of gateways: {ex}") - sys.exit(1) - -try: - num_checkers = int(sys.argv[3]) - if num_checkers < 1 or num_checkers > 250: - print("The number of checkers must be between 1 and 250") - sys.exit(1) -except Exception as ex: - print(f"Could not parse number of gateways: {ex}") - sys.exit(1) - -gateway_configs: List[GatewayConfig] = list() -for i in range(num_gateways): - wg_privkey, wg_pubkey = gen_wireguard_keys() - local_ip = ROUTER_IP_TEMPLATE % (i + 1) - gateway_configs.append(GatewayConfig( - private_key=wg_privkey, - public_key=wg_pubkey, - address=local_ip, - allowed_ips=[local_ip], - domain_name=GATEWAY_DOMAIN_TEMPLATE % (i + 1), - peers=[], - listen_port=GATEWAY_LISTEN_PORT, - )) - -elk_privkey, elk_pubkey = gen_wireguard_keys() -elk_config = GatewayConfig( - private_key=elk_privkey, - public_key=elk_pubkey, - address=ELK_IP_ADDRESS, - allowed_ips=[ELK_IP_ADDRESS], - domain_name=ELK_DOMAIN, - peers=[], - listen_port=ELK_LISTEN_PORT, -) -elk_peer = Peer( - public_key=elk_pubkey, - allowed_ips=[ELK_IP_ADDRESS], - endpoint=f"{ELK_DOMAIN}:{ELK_LISTEN_PORT}", -) - -engine_privkey, engine_pubkey = gen_wireguard_keys() -engine_config = GatewayConfig( - private_key=engine_privkey, - public_key=engine_pubkey, - address=ENGINE_IP_ADDRESS, - allowed_ips=[ENGINE_IP_ADDRESS], - domain_name=ENGINE_DOMAIN, - peers=[], - listen_port=ENGINE_LISTEN_PORT, -) -engine_peer = Peer( - public_key=engine_pubkey, - allowed_ips=[ENGINE_IP_ADDRESS], - endpoint=f"{ENGINE_DOMAIN}:{ENGINE_LISTEN_PORT}", -) - -elk_config.peers.append(engine_peer) -engine_config.peers.append(elk_peer) - -gateway_configs: List[GatewayConfig] = list() -for i in range(num_gateways): - wg_privkey, wg_pubkey = gen_wireguard_keys() - local_ip = ROUTER_IP_TEMPLATE % (i + 1) - gateway_configs.append(GatewayConfig( - private_key=wg_privkey, - public_key=wg_pubkey, - address=local_ip, - allowed_ips=[local_ip], - domain_name=GATEWAY_DOMAIN_TEMPLATE % (i + 1), - # don't use elk_peer and engine_peer, as those include the endpoint - # this will cause trouble when starting the gateway before the rest - peers=[ - Peer( - public_key=elk_pubkey, - allowed_ips=[ELK_IP_ADDRESS], - endpoint=None, - ), - Peer( - public_key=engine_pubkey, - allowed_ips=[ENGINE_IP_ADDRESS], - endpoint=None, - ), - ], - listen_port=GATEWAY_LISTEN_PORT, - )) - -# add team subnets to allowed ips for the gateways -for i in range(num_teams): - x, y = (i // 250) + 1, (i % 250) + 1 - gateway_id = i % num_gateways - gateway_config = gateway_configs[gateway_id] - gateway_config.allowed_ips.append(TEAM_IP_PREFIX_TEMPLATE % (x, y) + "0" + TEAM_IP_SUBNET_SIZE) - -checker_peer_list: List[Peer] = [ - elk_peer, - engine_peer, -] - -for i in range(num_gateways): - gateway_config = gateway_configs[i] - gw_peer = Peer( - public_key=gateway_config.public_key, - allowed_ips=gateway_config.allowed_ips, - endpoint=f"{gateway_config.domain_name}:{gateway_config.listen_port}", - ) - elk_config.peers.append(gw_peer) - engine_config.peers.append(gw_peer) - checker_peer_list.append(gw_peer) - -checker_configs = list() -for i in range(num_checkers): - wg_privkey, wg_pubkey = gen_wireguard_keys() - local_ip = CHECKER_IP_TEMPLATE % (i + 1) - checker_config = CheckerConfig( - private_key=wg_privkey, - public_key=wg_pubkey, - address=local_ip, - peers=checker_peer_list, - listen_port=None, - ) - checker_peer = Peer( - public_key=wg_pubkey, - allowed_ips=[local_ip], - endpoint=None, - ) - elk_config.peers.append(checker_peer) - engine_config.peers.append(checker_peer) - for gateway_config in gateway_configs: - gateway_config.peers.append(checker_peer) - checker_configs.append(checker_config) - -try: - os.mkdir("gateway_configs") -except FileExistsError: - pass - -try: - os.mkdir("checker_configs") -except FileExistsError: - pass - -# generate the config files -for i in range(num_gateways): - with open(os.path.join("gateway_configs", f"gateway{i + 1}.conf"), "w") as f: - f.write(create_config_file(gateway_configs[i])) - -for i in range(num_checkers): - with open(os.path.join("checker_configs", f"checker{i + 1}.conf"), "w") as f: - f.write(create_config_file(checker_configs[i])) - -with open("elk.conf", "w") as f: - f.write(create_config_file(elk_config)) - -with open("engine.conf", "w") as f: - f.write(create_config_file(engine_config)) - diff --git a/config/openvpn_configs/gen_configs.sh b/config/openvpn_configs/gen_configs.sh deleted file mode 100755 index e4c71da..0000000 --- a/config/openvpn_configs/gen_configs.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -set -x - -[ -f dh.pem ] || openssl dhparam -out dh.pem 2048 -mkdir -p zips -rm -f zips/*.zip - -for i in $(seq 1 $1); do - gateway_id="$((($i - 1) % $2 + 1))" - ( - rm -rf /tmp/openvpn-ca - mkdir /tmp/openvpn-ca - touch /tmp/openvpn-ca/vars - cd /tmp/openvpn-ca - easyrsa init-pki - cp /usr/share/easy-rsa/openssl-easyrsa.cnf /tmp/openvpn-ca/pki/ - cp -r /usr/share/easy-rsa/x509-types/ /tmp/openvpn-ca/ - echo "CA" | easyrsa build-ca nopass - echo "server" | easyrsa gen-req server nopass - echo "yes" | easyrsa sign-req server server - echo "client" | easyrsa gen-req client nopass - echo "yes" | easyrsa sign-req client client - ) - ( - [ -d "team$i" ] && rm -r "team$i" - mkdir "team$i" && cd "team$i" - openvpn --genkey --secret ta.key - cp /tmp/openvpn-ca/pki/ca.crt . - cp /tmp/openvpn-ca/pki/issued/server.crt . - cp /tmp/openvpn-ca/pki/private/server.key . - cp /tmp/openvpn-ca/pki/issued/client.crt . - cp /tmp/openvpn-ca/pki/private/client.key . - - TEAM_SUBNET_PREFIX="10.$((($i - 1) / 250 + 1)).$((($i - 1) % 250 + 1))" - REMOTE_ADDRESS="vpn${gateway_id}.bambi.ovh" - SERVER_PORT="$(printf '3%04d' $i)" - - cat <<EOF > team${i}.conf -port ${SERVER_PORT} -local 0.0.0.0 -proto udp -dev team${i} -dev-type tun - -mode server -tls-server -key-direction 0 -cipher AES-256-CBC -auth SHA256 - -topology subnet -ifconfig ${TEAM_SUBNET_PREFIX}.129 255.255.255.128 -push "topology subnet" -ifconfig-pool ${TEAM_SUBNET_PREFIX}.130 ${TEAM_SUBNET_PREFIX}.254 -route-gateway ${TEAM_SUBNET_PREFIX}.129 -push "route-gateway ${TEAM_SUBNET_PREFIX}.129" -push "route 10.0.0.0 255.0.0.0" - -keepalive 10 120 -persist-key -persist-tun -status /etc/openvpn/server/server.log -verb 3 - -duplicate-cn - -<ca> -$(cat ca.crt) -</ca> -<cert> -$(cat server.crt) -</cert> -<key> -$(cat server.key) -</key> -<tls-auth> -$(cat ta.key) -</tls-auth> -<dh> -$(cat ../dh.pem) -</dh> -EOF - - cat <<EOF > client.conf -proto udp -dev tun -remote ${REMOTE_ADDRESS} ${SERVER_PORT} -resolv-retry infinite -nobind - -tls-client -remote-cert-tls server -key-direction 1 -cipher AES-256-CBC -auth SHA256 - -keepalive 10 120 -verb 3 -pull - -<ca> -$(cat ca.crt) -</ca> -<tls-auth> -$(cat ta.key) -</tls-auth> -<cert> -$(cat client.crt) -</cert> -<key> -$(cat client.key) -</key> -EOF - zip "../zips/gateway${gateway_id}.zip" "team${i}.conf" - ) - rm -r /tmp/openvpn-ca -done \ No newline at end of file diff --git a/config/passwords/gen_passwords.sh b/config/passwords/gen_passwords.sh deleted file mode 100755 index 547e58d..0000000 --- a/config/passwords/gen_passwords.sh +++ /dev/null @@ -1 +0,0 @@ -for i in $(seq 1 $1); do openssl rand -hex 12 > ./team$i.txt; done diff --git a/config/wireguard_router/gen_keys.py b/config/wireguard_router/gen_keys.py deleted file mode 100755 index 4e31b7a..0000000 --- a/config/wireguard_router/gen_keys.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 - -import os -import subprocess -import sys -from dataclasses import dataclass -from typing import List, Optional - - -GAME_NETWORK = "10.0.0.0/8" -ROUTER_IP_TEMPLATE = "10.13.0.%d/32" -TEAM_IP_PREFIX_TEMPLATE = "10.%d.%d." -TEAM_IP_SUBNET_SIZE = "/24" -TEAM_IP_WG_SUBNET_SIZE = "/25" -GATEWAY_DOMAIN_TEMPLATE = "vpn%d.bambi.ovh" -GATEWAY_LISTEN_PORT = 51820 - - -@dataclass -class Peer: - public_key: str - allowed_ips: List[str] - endpoint: Optional[str] - - -@dataclass -class WireguardConfig: - private_key: str - public_key: str - address: str - peers: List[Peer] - listen_port: Optional[int] - - -@dataclass -class GatewayConfig(WireguardConfig): - allowed_ips: List[str] - domain_name: str - - -@dataclass -class TeamConfig(WireguardConfig): - pass - - -def gen_wireguard_keys(): - privkey_raw = subprocess.check_output(["wg", "genkey"]) - privkey = privkey_raw.strip().decode() - result = subprocess.run(["wg", "pubkey"], stdout=subprocess.PIPE, input=privkey_raw) - pubkey = result.stdout.strip().decode() - return privkey, pubkey - - -def create_interface_section(config: WireguardConfig): - raw = f"""[Interface] -Address = {config.address} -PrivateKey = {config.private_key} -""" - if config.listen_port is not None: - raw += f"ListenPort = {config.listen_port}\n" - return raw - - -def create_peer_section(peer: Peer): - allowed_ips = ", ".join(peer.allowed_ips) - raw = f"""[Peer] -PublicKey = {peer.public_key} -AllowedIPs = {allowed_ips} -""" - if peer.endpoint is not None: - raw += f"""PersistentKeepalive = 15 -Endpoint = {peer.endpoint} -""" - return raw - - -def create_config_file(config: WireguardConfig): - sections = [create_interface_section(config)] - for peer in config.peers: - sections.append(create_peer_section(peer)) - return "\n".join(sections) - - -if len(sys.argv) < 3: - print(f"Please specify a number of teams and gateways. Usage: {sys.argv[0]} <no of teams> <no of gateways>") - sys.exit(1) - -try: - num_teams = int(sys.argv[1]) - if num_teams < 1 or num_teams > 2000: - print("The number of teams must be between 1 and 2000.") - sys.exit(1) -except Exception as ex: - print(f"Could not parse number of teams: {ex}") - sys.exit(1) - -try: - num_gateways = int(sys.argv[2]) - if num_gateways < 1 or num_gateways > num_teams or num_gateways > 250: - print("The number of gateways must be between 1 and min(250, <no of teams>)") - sys.exit(1) -except Exception as ex: - print(f"Could not parse number of gateways: {ex}") - sys.exit(1) - -gateway_configs: List[GatewayConfig] = list() -for i in range(num_gateways): - wg_privkey, wg_pubkey = gen_wireguard_keys() - local_ip = ROUTER_IP_TEMPLATE % (i + 1) - gateway_configs.append(GatewayConfig( - private_key=wg_privkey, - public_key=wg_pubkey, - address=local_ip, - allowed_ips=[local_ip], - domain_name=GATEWAY_DOMAIN_TEMPLATE % (i + 1), - peers=[], - listen_port=GATEWAY_LISTEN_PORT, - )) - -team_configs = list() -for i in range(num_teams): - wg_privkey, wg_pubkey = gen_wireguard_keys() - x, y = (i // 250) + 1, (i % 250) + 1 - gateway_id = i % num_gateways - gateway_config = gateway_configs[gateway_id] - team_configs.append(TeamConfig( - private_key=wg_privkey, - public_key=wg_pubkey, - address=TEAM_IP_PREFIX_TEMPLATE % (x, y) + "1/32", - peers=[ - Peer( - public_key=gateway_config.public_key, - allowed_ips=[GAME_NETWORK], - endpoint=f"{gateway_config.domain_name}:{GATEWAY_LISTEN_PORT}", - ) - ], - listen_port=None, - )) - gateway_config.peers.append(Peer( - public_key=wg_pubkey, - allowed_ips=[TEAM_IP_PREFIX_TEMPLATE % (x, y) + "0" + TEAM_IP_WG_SUBNET_SIZE], - endpoint=None, - )) - gateway_config.allowed_ips.append(TEAM_IP_PREFIX_TEMPLATE % (x, y) + "0" + TEAM_IP_SUBNET_SIZE) - -# create the peer entries for the gateways -for i in range(num_gateways): - for j in range(num_gateways): - if i == j: - continue - gateway_i = gateway_configs[i] - gateway_j = gateway_configs[j] - gateway_i.peers.append(Peer( - public_key=gateway_j.public_key, - allowed_ips=gateway_j.allowed_ips, - endpoint=f"{gateway_j.domain_name}:{GATEWAY_LISTEN_PORT}", - )) - -try: - os.mkdir("gateway_configs") -except FileExistsError: - pass - -try: - os.mkdir("team_configs") -except FileExistsError: - pass - -# generate the config files -for i in range(num_gateways): - with open(os.path.join("gateway_configs", f"gateway{i + 1}.conf"), "w") as f: - f.write(create_config_file(gateway_configs[i])) - -for i in range(num_teams): - with open(os.path.join("team_configs", f"team{i + 1}.conf"), "w") as f: - f.write(create_config_file(team_configs[i])) - diff --git a/configgen/.gitignore b/configgen/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/configgen/README.md b/configgen/README.md new file mode 100644 index 0000000..e69de29 diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py new file mode 100644 index 0000000..ba0dd6a --- /dev/null +++ b/configgen/configgen/__init__.py @@ -0,0 +1,94 @@ +import argparse +import logging +import secrets +import shutil +from pathlib import Path + +from configgen.gen_wireguard_game import gen_wireguard_game +from configgen.gen_wireguard_internal import gen_wireguard_internal +from configgen.util import DATA_DIR + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__file__) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--teams", type=int, required=True) + parser.add_argument("--dns", type=str, required=True) + parser.add_argument("--routers", type=int, default=2) + parser.add_argument("--checkers", type=int, default=10) + args = parser.parse_args() + teams: int = args.teams + dns: str = args.dns + routers: int = args.routers + checkers: int = args.checkers + logger.info(f"Generating for {teams} teams, {routers} gws and {checkers} checkers") + prepare_directories(teams) + gen_wireguard_game(teams, dns, routers) + gen_wireguard_internal(teams, checkers, routers) + gen_passwords(teams) + gen_userdata_portal(teams) + + +def prepare_directories(teams: int) -> None: + Path(f"{DATA_DIR}/passwords/").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/wg_game/").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/wg_internal/").mkdir(parents=True, exist_ok=True) + shutil.rmtree(DATA_DIR / "export") + Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/export/ansible/routers").mkdir(parents=True, exist_ok=True) + for team in range(1, teams + 1): + Path(f"{DATA_DIR}/export/portal/team{team}").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/export/terraform/team{team}").mkdir( + parents=True, exist_ok=True + ) + Path(f"{DATA_DIR}/export/terraform/teams").mkdir(parents=True, exist_ok=True) + + +def gen_userdata_portal(teams: int) -> None: + for team in range(1, teams + 1): + user_data = gen_userdata(team, True) + Path(f"{DATA_DIR}/export/portal/team{team}/user_data.sh").write_text(user_data) + + +def gen_userdata(team: int, portal: bool) -> str: + password = Path(f"{DATA_DIR}/export/portal/team{team}/password.txt").read_text() + if portal: + wg = Path(f"{DATA_DIR}/export/portal/team{team}/game.conf").read_text() + else: + wg = Path(f"{DATA_DIR}/export/terraform/team{team}/game.conf").read_text() + return f"""#!/bin/sh +cat <<EOF >> /etc/wireguard/game.conf +{wg} +EOF + +for service in $(ls /services/); do + cd "/services/$service" + if [ -f "/services/$service/setup.sh" ]; then + /services/$service/setup.sh {team} + fi + docker compose up -d & +done + +cat <<EOF | passwd +{password} +{password} +EOF + +systemctl enable wg-quick@game +systemctl start wg-quick@game +""" + + +def gen_passwords(teams: int) -> None: + for team in range(1, teams + 1): + pw_file = Path(f"{DATA_DIR}/passwords/team{team}.txt") + if not pw_file.exists(): + pw_file.write_text(secrets.token_hex(32)) + export_file = Path(f"{DATA_DIR}/export/portal/team{team}/password.txt") + export_file.write_text(pw_file.read_text()) + + +def gen_openvpn() -> None: + pass diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py new file mode 100644 index 0000000..aeec4f2 --- /dev/null +++ b/configgen/configgen/gen_wireguard_game.py @@ -0,0 +1,157 @@ +import logging +from pathlib import Path +from typing import Optional, Tuple + +from configgen.util import ( + CIDR_GAME, + DATA_DIR, + TEAM_IP_PREFIX_TEMPLATE, + TEAM_IP_SUBNET, + TEAM_IP_WG_SUBNET, + WG_LISTEN_PORT_GAME, + Peer, + TeamConfig, + WireguardRouterConfig, + create_config_file, + gen_wg_priv, + gen_wg_pub, + get_router_cidr_game, + get_vulnbox_cidr, +) + +logger = logging.getLogger(__file__) + + +def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: + """ + Generates the game wireguard config files as needed, reusing keys (if present) + """ + router_configs: list[WireguardRouterConfig] = [] + team_portal_configs: list[TeamConfig] = [] + team_terraform_configs: list[TeamConfig] = [] + + # Generate gw configs + for router_id in range(1, routers + 1): + local_ip = get_router_cidr_game(router_id) + private_key, public_key = get_router_wg(router_id) + router_configs.append( + WireguardRouterConfig( + router_id=router_id, + private_key=private_key, + public_key=public_key, + cidr=local_ip, + responsible_ips=[local_ip], + peers=[], + listen_port=WG_LISTEN_PORT_GAME, + ) + ) + + # Generate team configs and add the peers to gw configs + for team in range(1, teams + 1): + private_key, public_key = get_team_wg(team) + x, y = (team // 250), (team % 250) + router_index = (team - 1) % routers + router_config = router_configs[router_index] + team_portal_configs.append(gen_team_config(private_key, public_key, team, router_config, dns)) + team_terraform_configs.append( + gen_team_config(private_key, public_key, team, router_config, None) + ) + + router_config.peers.append( + Peer( + public_key=public_key, + allowed_ips=[ + TEAM_IP_PREFIX_TEMPLATE % (x, y) + "0" + TEAM_IP_WG_SUBNET + ], + endpoint=None, + comment=f"team{team}", + ) + ) + router_config.responsible_ips.append( + TEAM_IP_PREFIX_TEMPLATE % (x, y) + "0" + TEAM_IP_SUBNET + ) + + for i in range(routers): + for j in range(routers): + if i == j: + continue + router_i = router_configs[i] + router_j = router_configs[j] + router_i.peers.append( + Peer( + public_key=router_j.public_key, + allowed_ips=router_j.responsible_ips, + endpoint=f"[[ROUTER_ADDRESS_{router_j.router_id}]]:{WG_LISTEN_PORT_GAME}", + comment=f"router{router_j.router_id}", + ) + ) + + for team_portal_config in team_portal_configs: + Path( + f"{DATA_DIR}/export/portal/team{team_portal_config.team_id}/game.conf" + ).write_text(create_config_file(team_portal_config)) + + for team_terraform_config in team_terraform_configs: + Path( + f"{DATA_DIR}/export/terraform/team{team_terraform_config.team_id}/game.conf" + ).write_text(create_config_file(team_terraform_config)) + + for router_config in router_configs: + Path( + f"{DATA_DIR}/export/ansible/routers/router{router_config.router_id}_game.conf" + ).write_text(create_config_file(router_config)) + + +def get_team_wg(team: int) -> Tuple[str, str]: + try: + priv = Path(f"{DATA_DIR}/wg_game/team{team}.key").read_text() + return priv, gen_wg_pub(priv) + except FileNotFoundError: + pass + + logger.debug(f"Generating fresh privkey for team{team}") + priv = gen_wg_priv() + Path(f"{DATA_DIR}/wg_game/team{team}.key").write_text(priv) + return priv, gen_wg_pub(priv) + + +def get_router_wg(gw: int) -> Tuple[str, str]: + try: + priv = Path(f"{DATA_DIR}/wg_game/router{gw}.key").read_text() + return priv, gen_wg_pub(priv) + except FileNotFoundError: + pass + + logger.debug(f"Generating fresh privkey for gw{gw}") + priv = gen_wg_priv() + Path(f"{DATA_DIR}/wg_game/router{gw}.key").write_text(priv) + return priv, gen_wg_pub(priv) + + +def gen_team_config( + private_key: str, + public_key: str, + team_id: int, + router_config: WireguardRouterConfig, + dns_suffix: Optional[str], +) -> TeamConfig: + if dns_suffix: + endpoint = f"router{router_config.router_id}.{dns_suffix}:{WG_LISTEN_PORT_GAME}" + else: + endpoint = f"[[ROUTER_ADDRESS_{router_config.router_id}]]:{WG_LISTEN_PORT_GAME}" + + return TeamConfig( + private_key=private_key, + public_key=public_key, + cidr=get_vulnbox_cidr(team_id), + peers=[ + Peer( + public_key=router_config.public_key, + allowed_ips=[CIDR_GAME], + endpoint=endpoint, + comment=f"router{router_config.router_id}", + ) + ], + listen_port=None, + team_id=team_id, + ) diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py new file mode 100644 index 0000000..f085ca7 --- /dev/null +++ b/configgen/configgen/gen_wireguard_internal.py @@ -0,0 +1,134 @@ +import logging +from pathlib import Path + +from configgen.util import ( + CIDR_ELK, + CIDR_ENGINE, + DATA_DIR, + WG_LISTEN_PORT_INTERNAL, + Peer, + WireguardCheckerConfig, + WireguardConfig, + WireguardRouterConfig, + create_config_file, + get_checker_cidr, + get_router_cidr_internal, + get_router_index, + get_team_cidr, + get_wg, +) + +logger = logging.getLogger(__file__) + + +def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: + """ + Generates the internal wireguard config files as needed, reusing keys (if present) + """ + engine_privkey, engine_pubkey = get_wg(Path("wg_internal/engine.key")) + engine_config = WireguardConfig( + private_key=engine_privkey, + public_key=engine_pubkey, + cidr=CIDR_ENGINE, + peers=[], + listen_port=WG_LISTEN_PORT_INTERNAL, + ) + engine_peer = Peer( + public_key=engine_pubkey, + allowed_ips=[CIDR_ENGINE], + endpoint=f"[[ENGINE_ADDRESS]]:{WG_LISTEN_PORT_INTERNAL}", + comment="engine", + ) + + elk_privkey, elk_pubkey = get_wg(Path("wg_internal/elk.key")) + elk_config = WireguardConfig( + private_key=elk_privkey, + public_key=elk_pubkey, + cidr=CIDR_ELK, + peers=[], + listen_port=WG_LISTEN_PORT_INTERNAL, + ) + elk_peer = Peer( + public_key=elk_pubkey, + allowed_ips=[CIDR_ELK], + endpoint=f"[[ELK_ADDRESS]]:{WG_LISTEN_PORT_INTERNAL}", + comment="elk", + ) + + elk_config.peers.append(engine_peer) + engine_config.peers.append(elk_peer) + router_configs: list[WireguardRouterConfig] = [] + checker_configs: list[WireguardCheckerConfig] = [] + checker_peer_list: list[Peer] = [elk_peer, engine_peer] + for router_id in range(1, routers + 1): + local_cidr = get_router_cidr_internal(router_id) + private_key, public_key = get_wg(Path(f"wg_internal/router{router_id}.key")) + router_configs.append( + WireguardRouterConfig( + router_id=router_id, + private_key=private_key, + public_key=public_key, + cidr=local_cidr, + responsible_ips=[local_cidr], + peers=[engine_peer, elk_peer], + listen_port=WG_LISTEN_PORT_INTERNAL, + ) + ) + + # Route traffic to teams through the correct router + for team in range(1, teams + 1): + router_config = router_configs[get_router_index(routers, team)] + router_config.responsible_ips.append(get_team_cidr(team)) + + # Add router peers to elk, engine and checkers + for router_config in router_configs: + peer = Peer( + public_key=router_config.public_key, + allowed_ips=router_config.responsible_ips, + endpoint=f"[[ROUTER_ADDRESS_{router_config.router_id}]]:{WG_LISTEN_PORT_INTERNAL}", + comment=f"router{router_config.router_id}", + ) + elk_config.peers.append(peer) + engine_config.peers.append(peer) + checker_peer_list.append(peer) + + for checker in range(1, checkers + 1): + private_key, public_key = get_wg(Path(f"wg_internal/checker{checker}.key")) + checker_cidr = get_checker_cidr(checker) + checker_config = WireguardCheckerConfig( + checker_id=checker, + private_key=private_key, + public_key=public_key, + cidr=checker_cidr, + peers=checker_peer_list, + listen_port=None, + ) + checker_peer = Peer( + public_key=public_key, + allowed_ips=[checker_cidr], + endpoint=None, + comment=f"checker{checker}", + ) + elk_config.peers.append(checker_peer) + engine_config.peers.append(checker_peer) + for gateway_config in router_configs: + gateway_config.peers.append(checker_peer) + checker_configs.append(checker_config) + + # Save all to disk + for router_config in router_configs: + Path( + f"{DATA_DIR}/export/ansible/routers/router{router_config.router_id}_internal.conf" + ).write_text(create_config_file(router_config)) + + for checker_config in checker_configs: + Path( + f"{DATA_DIR}/export/ansible/checkers/checker{checker_config.checker_id}.conf" + ).write_text(create_config_file(checker_config)) + + Path(f"{DATA_DIR}/export/ansible/engine.conf").write_text( + create_config_file(engine_config) + ) + Path(f"{DATA_DIR}/export/ansible/elk.conf").write_text( + create_config_file(elk_config) + ) diff --git a/configgen/configgen/util.py b/configgen/configgen/util.py new file mode 100644 index 0000000..2a408b0 --- /dev/null +++ b/configgen/configgen/util.py @@ -0,0 +1,144 @@ +import logging +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Tuple + +logger = logging.getLogger(__file__) +DATA_DIR = Path("../config") +CIDR_GAME = "10.0.0.0/8" +CIDR_ENGINE = "192.168.1.0/32" +CIDR_ELK = "192.168.3.0/32" +TEAM_IP_PREFIX_TEMPLATE = "10.%d.%d." +TEAM_IP_SUBNET = "/24" +TEAM_IP_WG_SUBNET = "/25" +WG_LISTEN_PORT_GAME = 51820 +WG_LISTEN_PORT_INTERNAL = 51821 + + +@dataclass +class Peer: + public_key: str + allowed_ips: list[str] + endpoint: Optional[str] + comment: Optional[str] + + +@dataclass +class WireguardConfig: + private_key: str + public_key: str + cidr: str + peers: list[Peer] + listen_port: Optional[int] + + +@dataclass +class WireguardRouterConfig(WireguardConfig): + router_id: int + responsible_ips: list[str] + + +@dataclass +class TeamConfig(WireguardConfig): + team_id: int + + +@dataclass +class WireguardCheckerConfig(WireguardConfig): + checker_id: int + + +def run_subprocess(args: list[str], input: Optional[str] = None) -> bytes: + return subprocess.run(args, stdout=subprocess.PIPE, input=input).stdout + + +def gen_wg_priv() -> str: + return run_subprocess(["wg", "genkey"]).strip().decode() + + +def gen_wg_pub(priv: str) -> str: + return run_subprocess(["wg", "pubkey"], input=priv.encode()).strip().decode() # type: ignore + # TODO fix typeshed? + + +def create_config_file(config: WireguardConfig) -> str: + sections = [create_interface_section(config)] + for peer in config.peers: + sections.append(create_peer_section(peer)) + return "\n".join(sections) + + +def create_interface_section(config: WireguardConfig) -> str: + raw = f"""[Interface] +Address = {config.cidr} +PrivateKey = {config.private_key} +""" + if config.listen_port is not None: + raw += f"ListenPort = {config.listen_port}\n" + return raw + + +def create_peer_section(peer: Peer) -> str: + allowed_ips = ", ".join(peer.allowed_ips) + raw = "" + if peer.comment: + raw += f"# {peer.comment}\n" + raw += f"""[Peer] +PublicKey = {peer.public_key} +AllowedIPs = {allowed_ips} +""" + if peer.endpoint is not None: + raw += f"""PersistentKeepalive = 15 +Endpoint = {peer.endpoint} +""" + return raw + + +def get_wg(path: Path) -> Tuple[str, str]: + relative_path = DATA_DIR / path + try: + priv = relative_path.read_text() + return priv, gen_wg_pub(priv) + except FileNotFoundError: + pass + + logger.debug(f"Generating fresh privkey for {path}") + priv = gen_wg_priv() + relative_path.write_text(priv) + return priv, gen_wg_pub(priv) + + +def _get_team_octets(team_id: int) -> Tuple[int, int]: + return (team_id // 250), (team_id % 250) + + +def get_vulnbox_cidr(team_id: int) -> str: + x, y = _get_team_octets(team_id) + return TEAM_IP_PREFIX_TEMPLATE % (x, y) + "1/32" + + +def get_vulnbox_ip(team_id: int) -> str: + x, y = _get_team_octets(team_id) + return TEAM_IP_PREFIX_TEMPLATE % (x, y) + "1" + + +def get_router_index(routers: int, team_id: int) -> int: + return (team_id - 1) % routers + + +def get_team_cidr(team_id: int) -> str: + x, y = _get_team_octets(team_id) + return f"10.{x}.{y}.0/24" + + +def get_checker_cidr(checker_id: int) -> str: + return f"192.168.1.{checker_id}/32" + + +def get_router_cidr_game(router_id: int) -> str: + return f"10.13.0.{router_id}/32" + + +def get_router_cidr_internal(router_id: int) -> str: + return f"192.168.0.{router_id}/32" diff --git a/configgen/poetry.lock b/configgen/poetry.lock new file mode 100644 index 0000000..9ddc5c6 --- /dev/null +++ b/configgen/poetry.lock @@ -0,0 +1,243 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "argparse" +version = "1.4.0" +description = "Python command-line parsing library" +optional = false +python-versions = "*" +files = [ + {file = "argparse-1.4.0-py2.py3-none-any.whl", hash = "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"}, + {file = "argparse-1.4.0.tar.gz", hash = "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4"}, +] + +[[package]] +name = "black" +version = "23.11.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-23.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911"}, + {file = "black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f"}, + {file = "black-23.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394"}, + {file = "black-23.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f"}, + {file = "black-23.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479"}, + {file = "black-23.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244"}, + {file = "black-23.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221"}, + {file = "black-23.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5"}, + {file = "black-23.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187"}, + {file = "black-23.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6"}, + {file = "black-23.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b"}, + {file = "black-23.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142"}, + {file = "black-23.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055"}, + {file = "black-23.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4"}, + {file = "black-23.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06"}, + {file = "black-23.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07"}, + {file = "black-23.11.0-py3-none-any.whl", hash = "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e"}, + {file = "black-23.11.0.tar.gz", hash = "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "isort" +version = "5.12.0" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, + {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, +] + +[package.extras] +colors = ["colorama (>=0.4.3)"] +pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] +plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + +[[package]] +name = "mypy" +version = "1.6.1" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e5012e5cc2ac628177eaac0e83d622b2dd499e28253d4107a08ecc59ede3fc2c"}, + {file = "mypy-1.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8fbb68711905f8912e5af474ca8b78d077447d8f3918997fecbf26943ff3cbb"}, + {file = "mypy-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a1ad938fee7d2d96ca666c77b7c494c3c5bd88dff792220e1afbebb2925b5e"}, + {file = "mypy-1.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b96ae2c1279d1065413965c607712006205a9ac541895004a1e0d4f281f2ff9f"}, + {file = "mypy-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:40b1844d2e8b232ed92e50a4bd11c48d2daa351f9deee6c194b83bf03e418b0c"}, + {file = "mypy-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81af8adaa5e3099469e7623436881eff6b3b06db5ef75e6f5b6d4871263547e5"}, + {file = "mypy-1.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8c223fa57cb154c7eab5156856c231c3f5eace1e0bed9b32a24696b7ba3c3245"}, + {file = "mypy-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8032e00ce71c3ceb93eeba63963b864bf635a18f6c0c12da6c13c450eedb183"}, + {file = "mypy-1.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c46b51de523817a0045b150ed11b56f9fff55f12b9edd0f3ed35b15a2809de0"}, + {file = "mypy-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:19f905bcfd9e167159b3d63ecd8cb5e696151c3e59a1742e79bc3bcb540c42c7"}, + {file = "mypy-1.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82e469518d3e9a321912955cc702d418773a2fd1e91c651280a1bda10622f02f"}, + {file = "mypy-1.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4473c22cc296425bbbce7e9429588e76e05bc7342da359d6520b6427bf76660"}, + {file = "mypy-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59a0d7d24dfb26729e0a068639a6ce3500e31d6655df8557156c51c1cb874ce7"}, + {file = "mypy-1.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfd13d47b29ed3bbaafaff7d8b21e90d827631afda134836962011acb5904b71"}, + {file = "mypy-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:eb4f18589d196a4cbe5290b435d135dee96567e07c2b2d43b5c4621b6501531a"}, + {file = "mypy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:41697773aa0bf53ff917aa077e2cde7aa50254f28750f9b88884acea38a16169"}, + {file = "mypy-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7274b0c57737bd3476d2229c6389b2ec9eefeb090bbaf77777e9d6b1b5a9d143"}, + {file = "mypy-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbaf4662e498c8c2e352da5f5bca5ab29d378895fa2d980630656178bd607c46"}, + {file = "mypy-1.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bb8ccb4724f7d8601938571bf3f24da0da791fe2db7be3d9e79849cb64e0ae85"}, + {file = "mypy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:68351911e85145f582b5aa6cd9ad666c8958bcae897a1bfda8f4940472463c45"}, + {file = "mypy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:49ae115da099dcc0922a7a895c1eec82c1518109ea5c162ed50e3b3594c71208"}, + {file = "mypy-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b27958f8c76bed8edaa63da0739d76e4e9ad4ed325c814f9b3851425582a3cd"}, + {file = "mypy-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925cd6a3b7b55dfba252b7c4561892311c5358c6b5a601847015a1ad4eb7d332"}, + {file = "mypy-1.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8f57e6b6927a49550da3d122f0cb983d400f843a8a82e65b3b380d3d7259468f"}, + {file = "mypy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a43ef1c8ddfdb9575691720b6352761f3f53d85f1b57d7745701041053deff30"}, + {file = "mypy-1.6.1-py3-none-any.whl", hash = "sha256:4cbe68ef919c28ea561165206a2dcb68591c50f3bcf777932323bc208d949cf1"}, + {file = "mypy-1.6.1.tar.gz", hash = "sha256:4d01c00d09a0be62a4ca3f933e315455bde83f37f892ba4b08ce92f3cf44bcc1"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pathspec" +version = "0.11.2" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, +] + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "ruff" +version = "0.1.5" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.5-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:32d47fc69261c21a4c48916f16ca272bf2f273eb635d91c65d5cd548bf1f3d96"}, + {file = "ruff-0.1.5-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:171276c1df6c07fa0597fb946139ced1c2978f4f0b8254f201281729981f3c17"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ef33cd0bb7316ca65649fc748acc1406dfa4da96a3d0cde6d52f2e866c7b39"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2c205827b3f8c13b4a432e9585750b93fd907986fe1aec62b2a02cf4401eee6"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb408e3a2ad8f6881d0f2e7ad70cddb3ed9f200eb3517a91a245bbe27101d379"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f20dc5e5905ddb407060ca27267c7174f532375c08076d1a953cf7bb016f5a24"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aafb9d2b671ed934998e881e2c0f5845a4295e84e719359c71c39a5363cccc91"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4894dddb476597a0ba4473d72a23151b8b3b0b5f958f2cf4d3f1c572cdb7af7"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a00a7ec893f665ed60008c70fe9eeb58d210e6b4d83ec6654a9904871f982a2a"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a8c11206b47f283cbda399a654fd0178d7a389e631f19f51da15cbe631480c5b"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fa29e67b3284b9a79b1a85ee66e293a94ac6b7bb068b307a8a373c3d343aa8ec"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9b97fd6da44d6cceb188147b68db69a5741fbc736465b5cea3928fdac0bc1aeb"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:721f4b9d3b4161df8dc9f09aa8562e39d14e55a4dbaa451a8e55bdc9590e20f4"}, + {file = "ruff-0.1.5-py3-none-win32.whl", hash = "sha256:f80c73bba6bc69e4fdc73b3991db0b546ce641bdcd5b07210b8ad6f64c79f1ab"}, + {file = "ruff-0.1.5-py3-none-win_amd64.whl", hash = "sha256:c21fe20ee7d76206d290a76271c1af7a5096bc4c73ab9383ed2ad35f852a0087"}, + {file = "ruff-0.1.5-py3-none-win_arm64.whl", hash = "sha256:82bfcb9927e88c1ed50f49ac6c9728dab3ea451212693fe40d08d314663e412f"}, + {file = "ruff-0.1.5.tar.gz", hash = "sha256:5cbec0ef2ae1748fb194f420fb03fb2c25c3258c86129af7172ff8f198f125ab"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "639380462f639404bedf08394e616af6fd19caf39f673d8795b0320e526bce7b" diff --git a/configgen/pyproject.toml b/configgen/pyproject.toml new file mode 100644 index 0000000..c746ab1 --- /dev/null +++ b/configgen/pyproject.toml @@ -0,0 +1,37 @@ +[tool.poetry] +name = "configgen" +version = "0.1.0" +description = "" +authors = ["Benedikt Radtke <benediktradtke@gmail.com>"] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +argparse = "^1.4.0" + + +[tool.poetry.group.dev.dependencies] +mypy = "^1.6.1" +ruff = "^0.1.5" +black = "^23.11.0" +isort = "^5.12.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +configgen = "configgen:main" + +[tool.mypy] +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true + +# Untyped Definitions and Calls +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true diff --git a/docker-compose.yml b/docker-compose.yml index 916bd5e..d610cca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,11 @@ version: '2' services: bambictf: - image: ghcr.io/enowars/bambictf:latest + #image: ghcr.io/enowars/bambictf:latest + build: . environment: - - HCLOUD_TOKEN=${HCLOUD_TOKEN} # for packer - - TF_VAR_HCLOUD_TOKEN=${HCLOUD_TOKEN} # for terraform - - TF_VAR_HETZNERDNS_TOKEN=${HETZNERDNS_TOKEN} # for terraform + - HCLOUD_TOKEN=${HCLOUD_TOKEN} # for packer + - TF_VAR_HCLOUD_TOKEN=${HCLOUD_TOKEN} # for terraform + - TF_VAR_HETZNERDNS_TOKEN=${HETZNERDNS_TOKEN} # for terraform volumes: - ".:/bambictf" diff --git a/terraform/bambictf.tf b/terraform/bambictf.tf index 834cdd3..a14e899 100644 --- a/terraform/bambictf.tf +++ b/terraform/bambictf.tf @@ -14,15 +14,14 @@ terraform { variable "HETZNERDNS_TOKEN" {} variable "hetznerdns_zone" {} +variable "hetznerdns_suffix" {} variable "HCLOUD_TOKEN" {} -variable "gateway_count" {} +variable "router_count" {} variable "checker_count" {} -variable "engine_count" {} -variable "elk_count" {} variable "vulnbox_count" {} -variable "gateway_type" {} +variable "router_type" {} variable "checker_type" {} variable "engine_type" {} variable "elk_type" {} @@ -30,9 +29,6 @@ variable "vulnbox_type" {} variable "home_location" {} -variable "vpn_floating_ip_only" {} -variable "internal_floating_ip_only" {} - provider "hcloud" { token = var.HCLOUD_TOKEN } @@ -45,306 +41,165 @@ data "hetznerdns_zone" "zone" { name = var.hetznerdns_zone } -locals { - internal_floating_ip_only = var.vpn_floating_ip_only || var.internal_floating_ip_only - router_count = var.vpn_floating_ip_only ? 0 : var.gateway_count - checker_count = local.internal_floating_ip_only ? 0 : var.checker_count - engine_count = local.internal_floating_ip_only ? 0 : var.engine_count - engine_vpn_count = var.vpn_floating_ip_only ? 0 : var.engine_count - elk_count = local.internal_floating_ip_only ? 0 : var.elk_count - elk_vpn_count = var.vpn_floating_ip_only ? 0 : var.elk_count - router_type = var.gateway_type - checker_type = var.checker_type - engine_type = var.engine_type - elk_type = var.elk_type - - # use this only when not managing vulnboxes via the EnoLandingPage - vulnbox_count = local.internal_floating_ip_only ? 0 : var.vulnbox_count - vulnbox_type = var.vulnbox_type - - location = var.home_location -} - -data "hcloud_ssh_keys" "all_keys" { - with_selector = "type=admin" -} - data "hcloud_image" "bambirouter" { - with_selector = local.router_count > 0 ? "type=bambirouter" : null - name = local.router_count > 0 ? null : "debian-10" - most_recent = true -} - -data "hcloud_image" "bambichecker" { - with_selector = local.checker_count > 0 ? "type=bambichecker" : null - name = local.checker_count > 0 ? null : "debian-10" + with_selector = var.router_count > 0 ? "type=bambirouter" : null + name = var.router_count > 0 ? null : "debian-10" most_recent = true } data "hcloud_image" "bambiengine" { - with_selector = local.engine_count > 0 ? "type=bambiengine" : null - name = local.engine_count > 0 ? null : "debian-10" + with_selector = "type=bambiengine" most_recent = true } -data "hcloud_image" "bambielk" { - with_selector = local.elk_count > 0 ? "type=bambielk" : null - name = local.elk_count > 0 ? null : "debian-10" +data "hcloud_image" "bambichecker" { + with_selector = var.checker_count > 0 ? "type=bambichecker" : null + name = var.checker_count > 0 ? null : "debian-10" most_recent = true } -data "hcloud_image" "vulnbox" { - with_selector = local.vulnbox_count > 0 ? "type=bambivulnbox" : null - name = local.vulnbox_count > 0 ? null : "debian-10" +data "hcloud_image" "bambivulnbox" { + with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null + name = var.vulnbox_count > 0 ? null : "debian-10" most_recent = true } -/*data "hcloud_floating_ip" "vpn" { - count = var.gateway_count - name = "gateway${count.index + 1}" -}*/ +data "hcloud_ssh_keys" "all_keys" { + with_selector = "type=admin" +} -resource "hcloud_floating_ip" "vpn_ip" { - count = var.gateway_count +resource "hcloud_floating_ip" "bambirouter_ip" { + count = var.router_count type = "ipv4" - name = "gateway${count.index + 1}" + name = "router${count.index + 1}" home_location = var.home_location } -resource "hetznerdns_record" "vpn_dns" { - count = var.gateway_count +resource "hcloud_floating_ip_assignment" "bambirouter_ipa" { + count = var.router_count + floating_ip_id = hcloud_floating_ip.bambirouter_ip[count.index].id + server_id = hcloud_server.bambirouter[count.index].id +} + +resource "hetznerdns_record" "bambirouter_dns" { + count = var.router_count zone_id = data.hetznerdns_zone.zone.id - name = "vpn${count.index + 1}" - value = hcloud_floating_ip.vpn_ip[count.index].ip_address + name = "router${count.index + 1}${var.hetznerdns_suffix}" + value = hcloud_floating_ip.bambirouter_ip[count.index].ip_address type = "A" ttl = 60 } -resource "hcloud_server" "router" { - count = local.router_count +resource "hcloud_server" "bambirouter" { + count = var.router_count name = "router${count.index + 1}" image = data.hcloud_image.bambirouter.id - location = local.location - server_type = local.router_type - - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = <<TERRAFORMEOF -#!/bin/bash - -cat > /etc/netplan/60-floating-ip.yaml <<EOF -network: - version: 2 - ethernets: - eth0: - addresses: - - ${hcloud_floating_ip.vpn_ip[count.index].ip_address}/32 -EOF -ip addr add ${hcloud_floating_ip.vpn_ip[count.index].ip_address}/32 dev eth0 - -#!/bin/sh -cat <<EOF >> /etc/wireguard/internal.conf -${file("../config/internal_router/gateway_configs/gateway${count.index + 1}.conf")} -EOF -systemctl enable wg-quick@internal -systemctl start wg-quick@internal - -#!/bin/sh -cat <<EOF >> /etc/wireguard/router.conf -${file("../config/wireguard_router/gateway_configs/gateway${count.index + 1}.conf")} -EOF -systemctl enable wg-quick@router -systemctl start wg-quick@router - -( - cd /etc/openvpn/server/ - unzip /root/zips/gateway${count.index + 1}.zip - for i in $(ls *.conf); - do sed -i "s/local 0.0.0.0/local ${hcloud_floating_ip.vpn_ip[count.index].ip_address}/" $${i}; - name="openvpn-server@$(echo $i | sed 's/.conf//')"; - systemctl enable $name; - systemctl start $name; - done -) -TERRAFORMEOF + location = var.home_location + server_type = var.router_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_router.tftpl", { + index = count.index, + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = hcloud_floating_ip.bambielk_ip, + engine = hcloud_floating_ip.bambiengine_ip, + } + ) } -resource "hcloud_floating_ip_assignment" "vpn" { - count = local.router_count - floating_ip_id = hcloud_floating_ip.vpn_ip[count.index].id - server_id = hcloud_server.router[count.index].id +resource "hcloud_floating_ip" "bambiengine_ip" { + type = "ipv4" + name = "engine" + home_location = var.home_location } -resource "hcloud_server" "checker" { - name = "checker${count.index + 1}" - image = data.hcloud_image.bambichecker.id - location = local.location - server_type = local.checker_type - count = local.checker_count - - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - # ensure that the wireguard endpoint is resolved correctly on boot - depends_on = [ - hcloud_floating_ip.engine_vpn, - hcloud_floating_ip.elk_vpn, - hetznerdns_record.engine_vpn_dns, - hetznerdns_record.elk_vpn_dns, - hcloud_floating_ip.vpn_ip, - hetznerdns_record.vpn_dns, - ] - - user_data = <<TERRAFORMEOF -#!/bin/sh -cat <<EOF >> /etc/wireguard/internal.conf -${file("../config/internal_router/checker_configs/checker${count.index + 1}.conf")} -EOF -systemctl enable wg-quick@internal -systemctl start wg-quick@internal - -for service in $(ls /services/); do -cd "/services/$service" && docker-compose up -d & -done -TERRAFORMEOF +resource "hcloud_floating_ip_assignment" "bambiengine_ipa" { + floating_ip_id = hcloud_floating_ip.bambiengine_ip.id + server_id = hcloud_server.bambiengine.id } -resource "hetznerdns_record" "checker_dns" { - count = local.checker_count +resource "hetznerdns_record" "bambiengine_dns" { zone_id = data.hetznerdns_zone.zone.id - name = "checker${count.index + 1}" - value = hcloud_server.checker[count.index].ipv4_address + name = "engine${var.hetznerdns_suffix}" + value = hcloud_floating_ip.bambiengine_ip.ip_address type = "A" ttl = 60 } -resource "hcloud_server" "engine" { - name = "engine" +resource "hcloud_server" "bambiengine" { + name = "bambiengine" image = data.hcloud_image.bambiengine.id - location = local.location - server_type = local.engine_type - count = local.engine_count - - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - # ensure that the wireguard endpoint is resolved correctly on boot - depends_on = [ - hcloud_floating_ip.elk_vpn, - hetznerdns_record.elk_vpn_dns, - hcloud_floating_ip.vpn_ip, - hetznerdns_record.vpn_dns, - ] - - user_data = <<TERRAFORMEOF -#!/bin/sh -cat > /etc/netplan/60-floating-ip.yaml <<EOF -network: - version: 2 - ethernets: - eth0: - addresses: - - ${hcloud_floating_ip.engine_vpn[0].ip_address}/32 -EOF -ip addr add ${hcloud_floating_ip.engine_vpn[0].ip_address}/32 dev eth0 - -cat <<EOF >> /etc/wireguard/internal.conf -${file("../config/internal_router/engine.conf")} -EOF -systemctl enable wg-quick@internal -systemctl start wg-quick@internal -TERRAFORMEOF -} - -resource "hcloud_server" "elk" { - name = "elk" - image = data.hcloud_image.bambielk.id - location = local.location - server_type = local.elk_type - count = local.elk_count - - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = <<TERRAFORMEOF -#!/bin/sh -cd "/services/EnoELK" && docker-compose up -d & -cat > /etc/netplan/60-floating-ip.yaml <<EOF -network: - version: 2 - ethernets: - eth0: - addresses: - - ${hcloud_floating_ip.elk_vpn[0].ip_address}/32 -EOF -ip addr add ${hcloud_floating_ip.elk_vpn[0].ip_address}/32 dev eth0 - -cat <<EOF >> /etc/wireguard/internal.conf -${file("../config/internal_router/elk.conf")} -EOF -systemctl enable wg-quick@internal -systemctl start wg-quick@internal -TERRAFORMEOF + location = var.home_location + server_type = var.engine_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_engine.tftpl", { + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = hcloud_floating_ip.bambielk_ip, + engine = hcloud_floating_ip.bambiengine_ip, + } + ) } -resource "hcloud_floating_ip" "engine_vpn" { - count = local.engine_vpn_count - name = "engine-vpn" - type = "ipv4" - home_location = local.location +resource "hcloud_server" "bambichecker" { + name = "bambichecker" + image = data.hcloud_image.bambichecker.id + location = var.home_location + server_type = var.checker_type + count = var.checker_count + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_checker.tftpl", { + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = hcloud_floating_ip.bambielk_ip, + engine = hcloud_floating_ip.bambiengine_ip, + } + ) } -resource "hetznerdns_record" "engine_vpn_dns" { - count = local.engine_vpn_count +resource "hetznerdns_record" "bambchecker_dns" { + count = var.checker_count zone_id = data.hetznerdns_zone.zone.id - name = "engine-vpn" - value = hcloud_floating_ip.engine_vpn[count.index].ip_address + name = "checker${count.index + 1}${var.hetznerdns_suffix}" + value = hcloud_server.bambichecker[count.index].ipv4_address type = "A" ttl = 60 } -resource "hcloud_floating_ip_assignment" "engine_vpn" { - count = local.engine_count - floating_ip_id = hcloud_floating_ip.engine_vpn[0].id - server_id = hcloud_server.engine[0].id -} - -resource "hcloud_floating_ip" "elk_vpn" { - count = local.elk_vpn_count - name = "elk-vpn" +resource "hcloud_floating_ip" "bambielk_ip" { type = "ipv4" - home_location = local.location -} - -resource "hcloud_floating_ip_assignment" "elk_vpn" { - count = local.elk_count - floating_ip_id = hcloud_floating_ip.elk_vpn[0].id - server_id = hcloud_server.elk[0].id -} - -resource "hetznerdns_record" "elk_vpn_dns" { - count = local.elk_vpn_count - zone_id = data.hetznerdns_zone.zone.id - name = "elk-vpn" - value = hcloud_floating_ip.elk_vpn[count.index].ip_address - type = "A" - ttl = 60 + name = "elk" + home_location = var.home_location } -resource "hcloud_server" "vulnbox" { +resource "hcloud_server" "bambivulnbox" { name = "vulnbox${count.index + 1}" - image = data.hcloud_image.vulnbox.id - location = local.location - server_type = local.vulnbox_type - count = local.vulnbox_count - + image = data.hcloud_image.bambivulnbox.id + location = var.home_location + server_type = var.vulnbox_type + count = var.vulnbox_count ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - user_data = file("../config/export/team${count.index + 1}/user_data.sh") + user_data = templatefile( + "user_data_vulnbox.tftpl", { + wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), + index = count.index, + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip + } + ) } -resource "hetznerdns_record" "vulnbox_dns" { - count = local.vulnbox_count +resource "hetznerdns_record" "bambivulnbox_dns" { + count = var.vulnbox_count zone_id = data.hetznerdns_zone.zone.id - name = "vulnbox${count.index + 1}" - value = hcloud_server.vulnbox[count.index].ipv4_address + name = "team${count.index + 1}${var.hetznerdns_suffix}" + value = hcloud_server.bambivulnbox[count.index].ipv4_address type = "A" ttl = 60 } diff --git a/terraform/terraform.tfvars.sample b/terraform/terraform.tfvars.sample index fbd6784..3c83c16 100644 --- a/terraform/terraform.tfvars.sample +++ b/terraform/terraform.tfvars.sample @@ -1,7 +1,8 @@ -hetznerdns_zone = "bambi.ovh" -home_location = "fsn1" +hetznerdns_zone = "bambi.ovh" +hetznerdns_suffix = "prod.bambi.ovh" +home_location = "fsn1" -gateway_count = 2 +router_count = 2 checker_count = 1 engine_count = 1 elk_count = 1 @@ -14,4 +15,4 @@ elk_type = "cpx31" vulnbox_type = "cpx21" vpn_floating_ip_only = true -internal_floating_ip_only = true \ No newline at end of file +internal_floating_ip_only = true diff --git a/terraform/user_data_checker.tftpl b/terraform/user_data_checker.tftpl new file mode 100644 index 0000000..90c065c --- /dev/null +++ b/terraform/user_data_checker.tftpl @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +# Network +mv "/etc/wireguard/checker${id}.conf" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine.ip_address}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk.ip_address}#g" "/etc/wireguard/internal.conf" +%{ for router_index, router in router_ips ~} + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" +%{ endfor ~} +systemctl enable "wg-quick@internal" +systemctl start "wg-quick@internal" + +# Checkers +for service in $(ls /services/); do + cd "/services/$service" + docker compose up -d & +done diff --git a/terraform/user_data_engine.tftpl b/terraform/user_data_engine.tftpl new file mode 100644 index 0000000..4bc686f --- /dev/null +++ b/terraform/user_data_engine.tftpl @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +cat > /etc/netplan/60-floating-ip.yaml <<EOF +network: + version: 2 + ethernets: + eth0: + addresses: + - ${engine.ip_address}/32 +EOF +netplan apply + +%{ for router_index, router in router_ips ~} + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" +%{ endfor ~} + +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk.ip_address}#g" "/etc/wireguard/internal.conf" + +systemctl enable "wg-quick@internal" +systemctl start "wg-quick@internal" diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl new file mode 100644 index 0000000..7c95430 --- /dev/null +++ b/terraform/user_data_router.tftpl @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +cat > /etc/netplan/60-floating-ip.yaml <<EOF +network: + version: 2 + ethernets: + eth0: + addresses: + - ${router_ips[index].ip_address}/32 +EOF +netplan apply + +%{ for router_index, router in router_ips ~} + %{ if router_index != index } + rm "/etc/wireguard/router${router_index+1}_internal.conf" + rm "/etc/wireguard/router${router_index+1}_game.conf" + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/router${id}_internal.conf" + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/router${id}_game.conf" + %{ endif } +%{ endfor ~} + +mv "/etc/wireguard/router${id}_internal.conf" "/etc/wireguard/internal.conf" +mv "/etc/wireguard/router${id}_game.conf" "/etc/wireguard/router.conf" + +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine.ip_address}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk.ip_address}#g" "/etc/wireguard/internal.conf" + +systemctl enable "wg-quick@internal" +systemctl start "wg-quick@internal" +systemctl enable "wg-quick@router" +systemctl start "wg-quick@router" diff --git a/terraform/user_data_vulnbox.tftpl b/terraform/user_data_vulnbox.tftpl new file mode 100644 index 0000000..9688996 --- /dev/null +++ b/terraform/user_data_vulnbox.tftpl @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +# Network +cat > /etc/wireguard/game.conf <<EOF +${wgconf} +EOF +%{ for router_index, router in router_ips ~} + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/game.conf" +%{ endfor ~} +systemctl enable "wg-quick@game" +systemctl start "wg-quick@game" + +# Services +for service in $(ls /services/); do + cd "/services/$service" + if [ -f "/services/$service/setup.sh" ]; then + /services/$service/setup.sh 2 + fi + docker compose up -d & +done From 3a7636b99c82df74e7f9dc9fc22a4f5f462efd3e Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 16 Nov 2023 11:43:37 +0100 Subject: [PATCH 02/61] use new configgen in CI --- .github/workflows/packer.yaml | 4 ++-- Dockerfile | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packer.yaml b/.github/workflows/packer.yaml index 581eef3..2dff535 100644 --- a/.github/workflows/packer.yaml +++ b/.github/workflows/packer.yaml @@ -53,6 +53,6 @@ jobs: - name: prepare ansible config run: docker-compose exec -T bambictf sh -c 'cp ansible/config_bambi.yml.sample ansible/config_bambi.yml' - name: generate config files - run: docker-compose exec -T bambictf sh -c 'cd config; TEAM_COUNT=4 GATEWAY_COUNT=2 CHECKER_COUNT=2 ./gen_config.sh' + run: docker-compose exec -T bambictf sh -c 'cd configgen; poetry install; poetry run configgen --teams 4 --routers 2 --dns bambi.ovh' - name: build packer image - run: docker-compose exec -T bambictf sh -c 'cd packer; packer build ${{ matrix.image }}.json' \ No newline at end of file + run: docker-compose exec -T bambictf sh -c 'cd packer; packer build ${{ matrix.image }}.json' diff --git a/Dockerfile b/Dockerfile index 6fd1fae..068790f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tz RUN apt-get install -y --no-install-recommends rsync git less tmux python3 curl wireguard python3-pip unzip file nano dnsutils jq \ software-properties-common gpg-agent # for ansible and packer install +# Poetry +RUN pip install poetry && poetry config settings.virtualenvs.in-project true + # Ansible RUN add-apt-repository --yes --update ppa:ansible/ansible && apt-get install -y ansible From 94c87bd7697fa469e7fcbca4e1ba8f68757db1a3 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 16 Nov 2023 11:46:44 +0100 Subject: [PATCH 03/61] use correct config prefix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 068790f..cabe44e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get install -y --no-install-recommends rsync git less tmux python3 curl software-properties-common gpg-agent # for ansible and packer install # Poetry -RUN pip install poetry && poetry config settings.virtualenvs.in-project true +RUN pip install poetry && poetry config virtualenvs.in-project true # Ansible RUN add-apt-repository --yes --update ppa:ansible/ansible && apt-get install -y ansible From fa0f58e5c38368cbd35ab88ecec67f145df1a603 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 16 Nov 2023 12:10:02 +0100 Subject: [PATCH 04/61] don't fail if export does not exist --- configgen/configgen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index ba0dd6a..41faa06 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -35,7 +35,7 @@ def prepare_directories(teams: int) -> None: Path(f"{DATA_DIR}/passwords/").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/wg_game/").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/wg_internal/").mkdir(parents=True, exist_ok=True) - shutil.rmtree(DATA_DIR / "export") + shutil.rmtree(DATA_DIR / "export", ignore_errors=True) Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/routers").mkdir(parents=True, exist_ok=True) for team in range(1, teams + 1): From f26f7982101429a8c842c1adc9e6ad81a472ec02 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 16 Nov 2023 12:52:30 +0100 Subject: [PATCH 05/61] fix elk builds --- ansible/roles/enoelk/meta/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/roles/enoelk/meta/main.yml b/ansible/roles/enoelk/meta/main.yml index bf5ac38..c1f68fc 100644 --- a/ansible/roles/enoelk/meta/main.yml +++ b/ansible/roles/enoelk/meta/main.yml @@ -1,3 +1,2 @@ dependencies: - role: "docker" -- role: "metricbeat" From 4ae7d6a57c90e9740608f1a91173626802803299 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 16 Nov 2023 22:08:38 +0100 Subject: [PATCH 06/61] elk --- ansible/bambielk.yml | 3 +- ansible/roles/wireguard_elk/tasks/main.yml | 5 +++ configgen/configgen/gen_wireguard_game.py | 4 +- configgen/pyproject.toml | 3 ++ terraform/.terraform.tfstate.lock.info | 1 + terraform/bambictf.tf | 52 ++++++++++++++++++---- terraform/user_data_elk.tftpl | 27 +++++++++++ 7 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 ansible/roles/wireguard_elk/tasks/main.yml create mode 100644 terraform/.terraform.tfstate.lock.info create mode 100644 terraform/user_data_elk.tftpl diff --git a/ansible/bambielk.yml b/ansible/bambielk.yml index a6f3557..a08e80a 100644 --- a/ansible/bambielk.yml +++ b/ansible/bambielk.yml @@ -11,6 +11,7 @@ - wireguard - enoelk - docker-block-external + - wireguard_elk - role: filebeat vars: elk: 192.168.3.0 @@ -24,4 +25,4 @@ vars: program_list: - "tmux" - - "git" \ No newline at end of file + - "git" diff --git a/ansible/roles/wireguard_elk/tasks/main.yml b/ansible/roles/wireguard_elk/tasks/main.yml new file mode 100644 index 0000000..b6ced41 --- /dev/null +++ b/ansible/roles/wireguard_elk/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/elk.conf" + dest: /etc/wireguard/internal.conf diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index aeec4f2..c37c540 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -52,7 +52,9 @@ def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: x, y = (team // 250), (team % 250) router_index = (team - 1) % routers router_config = router_configs[router_index] - team_portal_configs.append(gen_team_config(private_key, public_key, team, router_config, dns)) + team_portal_configs.append( + gen_team_config(private_key, public_key, team, router_config, dns) + ) team_terraform_configs.append( gen_team_config(private_key, public_key, team, router_config, None) ) diff --git a/configgen/pyproject.toml b/configgen/pyproject.toml index c746ab1..0130779 100644 --- a/configgen/pyproject.toml +++ b/configgen/pyproject.toml @@ -35,3 +35,6 @@ disallow_untyped_defs = true disallow_incomplete_defs = true check_untyped_defs = true disallow_untyped_decorators = true + +[tool.isort] +profile = "black" diff --git a/terraform/.terraform.tfstate.lock.info b/terraform/.terraform.tfstate.lock.info new file mode 100644 index 0000000..e3a9d48 --- /dev/null +++ b/terraform/.terraform.tfstate.lock.info @@ -0,0 +1 @@ +{"ID":"2da0948e-8d55-f4f6-b253-f9d2c1498ba9","Operation":"OperationTypeApply","Info":"","Who":"root@d9db1bc35b27","Version":"1.0.11","Created":"2023-11-16T21:08:30.177346053Z","Path":"terraform.tfstate"} \ No newline at end of file diff --git a/terraform/bambictf.tf b/terraform/bambictf.tf index a14e899..93734b6 100644 --- a/terraform/bambictf.tf +++ b/terraform/bambictf.tf @@ -52,6 +52,11 @@ data "hcloud_image" "bambiengine" { most_recent = true } +data "hcloud_image" "bambielk" { + with_selector = "type=bambielk" + most_recent = true +} + data "hcloud_image" "bambichecker" { with_selector = var.checker_count > 0 ? "type=bambichecker" : null name = var.checker_count > 0 ? null : "debian-10" @@ -144,6 +149,15 @@ resource "hcloud_server" "bambiengine" { ) } +resource "hetznerdns_record" "bambchecker_dns" { + count = var.checker_count + zone_id = data.hetznerdns_zone.zone.id + name = "checker${count.index + 1}${var.hetznerdns_suffix}" + value = hcloud_server.bambichecker[count.index].ipv4_address + type = "A" + ttl = 60 +} + resource "hcloud_server" "bambichecker" { name = "bambichecker" image = data.hcloud_image.bambichecker.id @@ -162,19 +176,39 @@ resource "hcloud_server" "bambichecker" { ) } -resource "hetznerdns_record" "bambchecker_dns" { - count = var.checker_count +resource "hcloud_floating_ip" "bambielk_ip" { + type = "ipv4" + name = "elk" + home_location = var.home_location +} + +resource "hcloud_floating_ip_assignment" "bambielk_ipa" { + floating_ip_id = hcloud_floating_ip.bambielk_ip.id + server_id = hcloud_server.bambielk.id +} + +resource "hetznerdns_record" "bambielk_dns" { zone_id = data.hetznerdns_zone.zone.id - name = "checker${count.index + 1}${var.hetznerdns_suffix}" - value = hcloud_server.bambichecker[count.index].ipv4_address + name = "elk${var.hetznerdns_suffix}" + value = hcloud_server.bambielk.ipv4_address type = "A" ttl = 60 } -resource "hcloud_floating_ip" "bambielk_ip" { - type = "ipv4" - name = "elk" - home_location = var.home_location +resource "hcloud_server" "bambielk" { + name = "bambielk" + image = data.hcloud_image.bambielk.id + location = var.home_location + server_type = var.elk_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_elk.tftpl", { + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = hcloud_floating_ip.bambielk_ip, + engine = hcloud_floating_ip.bambiengine_ip, + } + ) } resource "hcloud_server" "bambivulnbox" { @@ -183,7 +217,7 @@ resource "hcloud_server" "bambivulnbox" { location = var.home_location server_type = var.vulnbox_type count = var.vulnbox_count - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id user_data = templatefile( "user_data_vulnbox.tftpl", { diff --git a/terraform/user_data_elk.tftpl b/terraform/user_data_elk.tftpl new file mode 100644 index 0000000..fbf210f --- /dev/null +++ b/terraform/user_data_elk.tftpl @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +# Network +cat > /etc/netplan/60-floating-ip.yaml <<EOF +network: + version: 2 + ethernets: + eth0: + addresses: + - ${elk.ip_address}/32 +EOF +netplan apply + +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine.ip_address}#g" "/etc/wireguard/internal.conf" +%{ for router_index, router in router_ips ~} + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" +%{ endfor ~} +systemctl enable "wg-quick@internal" +systemctl start "wg-quick@internal" + +# Checkers +for service in $(ls /services/); do + cd "/services/$service" + docker compose up -d & +done From 98d3f9d1b6231031615cf28ae27231f7eca12f77 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 17 Nov 2023 19:13:42 +0100 Subject: [PATCH 07/61] =?UTF-8?q?w=C3=B6rkw=C3=B6rk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/packer.yaml | 10 +-- ansible/roles/docker/tasks/main.yml | 86 +++++----------------- ansible/roles/dotnetsdk/tasks/main.yml | 27 +------ ansible/roles/enoelk/tasks/main.yml | 2 +- ansible/roles/enoengine/tasks/main.yml | 4 +- ansible/roles/vuln_checkers/tasks/main.yml | 7 +- ansible/roles/vuln_services/tasks/main.yml | 7 +- terraform/.terraform.tfstate.lock.info | 1 - terraform/bambictf.tf | 58 +++++---------- terraform/bambivulnbox.tf | 36 +++++++++ 10 files changed, 91 insertions(+), 147 deletions(-) delete mode 100644 terraform/.terraform.tfstate.lock.info create mode 100644 terraform/bambivulnbox.tf diff --git a/.github/workflows/packer.yaml b/.github/workflows/packer.yaml index 2dff535..9b057c5 100644 --- a/.github/workflows/packer.yaml +++ b/.github/workflows/packer.yaml @@ -46,13 +46,13 @@ jobs: path: /tmp - name: Load Docker image run: docker load --input /tmp/bambictf.tar - - name: start docker-compose setup + - name: start docker compose setup env: HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - run: docker-compose up -d + run: docker compose up -d - name: prepare ansible config - run: docker-compose exec -T bambictf sh -c 'cp ansible/config_bambi.yml.sample ansible/config_bambi.yml' + run: docker compose exec -T bambictf sh -c 'cp ansible/config_bambi.yml.sample ansible/config_bambi.yml' - name: generate config files - run: docker-compose exec -T bambictf sh -c 'cd configgen; poetry install; poetry run configgen --teams 4 --routers 2 --dns bambi.ovh' + run: docker compose exec -T bambictf sh -c 'cd configgen; poetry install; poetry run configgen --teams 4 --routers 2 --dns bambi.ovh' - name: build packer image - run: docker-compose exec -T bambictf sh -c 'cd packer; packer build ${{ matrix.image }}.json' + run: docker compose exec -T bambictf sh -c 'cd packer; packer build ${{ matrix.image }}.json' diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index 7bcee6f..e29fb8f 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -1,75 +1,23 @@ --- - - name: Get release name - command: lsb_release -cs - register: release - changed_when: false - - - name: Get OS name - command: uname -s - register: uns - changed_when: false - - - name: Get OS arch - command: uname -m - register: unm - changed_when: false - - - name: Install docker dependencies - apt: - name: ["apt-transport-https", "ca-certificates", "gnupg2", "software-properties-common"] - state: present - - - name: Copy over the docker gpg key - copy: - src: docker.gpg - dest: /tmp/ - changed_when: false - - - name: Import Docker GPG key - apt_key: - file: /tmp/docker.gpg - state: present - - - name: Delete the docker key - file: - path: /tmp/docker.gpg - state: absent - changed_when: false - - - name: Add docker repository - apt_repository: - repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ release.stdout }} stable" - state: present - - - name: Update and install docker - apt: - name: docker-ce - state: present - update_cache: yes - - - name: Install docker-compose - get_url: - url: "https://github.com/docker/compose/releases/download/1.29.0/docker-compose-{{ uns.stdout }}-{{ unm.stdout }}" - dest: /usr/local/bin/docker-compose - mode: 0755 - - - name: Create /services - file: - path: /services - state: directory + # https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository + - name: Install docker + shell: | + apt-get update -y + apt-get install -y ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + + # Add the repository to Apt sources: + echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get -y update + + apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - name: Copy daemon.json copy: src: daemon.json dest: /etc/docker/daemon.json - - - name: Restart docker - service: - name: docker - state: restarted - enabled: yes - - - name: Reload system daemon - systemd: - name: "docker" # ansible <2.4 always requires 'name' - daemon_reload: yes diff --git a/ansible/roles/dotnetsdk/tasks/main.yml b/ansible/roles/dotnetsdk/tasks/main.yml index 70c8082..9360467 100644 --- a/ansible/roles/dotnetsdk/tasks/main.yml +++ b/ansible/roles/dotnetsdk/tasks/main.yml @@ -1,31 +1,6 @@ --- - - name: Check if dotnet-sdk is installed - command: dpkg-query -W dotnet-sdk - register: dotnet_sdk_deb - failed_when: dotnet_sdk_deb.rc > 1 - changed_when: dotnet_sdk_deb.rc == 1 - - # - name: Download Microsoft PGP key - # get_url: - # url: "https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb" - # dest: "/tmp/packages-microsoft-prod.deb" - # when: dotnet_sdk_deb.rc == 1 - - # - name: Install Microsoft PGP key - # apt: - # deb: "/tmp/packages-microsoft-prod.deb" - # become: yes - # when: dotnet_sdk_deb.rc == 1 - - # - name: Install dotnet deps - # apt: - # name: "apt-transport-https" - # state: present - # when: dotnet_sdk_deb.rc == 1 - - name: Install dotnet sdk apt: name: "dotnet-sdk-6.0" state: present - update_cache: yes - when: dotnet_sdk_deb.rc == 1 + cache_valid_time: 250 diff --git a/ansible/roles/enoelk/tasks/main.yml b/ansible/roles/enoelk/tasks/main.yml index 2e1cd56..19f59e3 100644 --- a/ansible/roles/enoelk/tasks/main.yml +++ b/ansible/roles/enoelk/tasks/main.yml @@ -16,7 +16,7 @@ dest: /services/EnoELK - name: Pull ELK images - shell: docker-compose pull + shell: docker compose pull args: chdir: /services/EnoELK diff --git a/ansible/roles/enoengine/tasks/main.yml b/ansible/roles/enoengine/tasks/main.yml index 953d8b2..fec2db0 100644 --- a/ansible/roles/enoengine/tasks/main.yml +++ b/ansible/roles/enoengine/tasks/main.yml @@ -15,7 +15,7 @@ chdir: /services/EnoEngine - name: Start EnoEngine Postgres - shell: "docker-compose up -d" + shell: "docker compose up -d" args: chdir: /services/EnoEngine @@ -31,7 +31,7 @@ changed_when: false - name: Start EnoCTFPortal - shell: "docker-compose up -d" + shell: "docker compose up -d" args: chdir: /services/EnoCTFPortal diff --git a/ansible/roles/vuln_checkers/tasks/main.yml b/ansible/roles/vuln_checkers/tasks/main.yml index 9fdd190..2518ff6 100644 --- a/ansible/roles/vuln_checkers/tasks/main.yml +++ b/ansible/roles/vuln_checkers/tasks/main.yml @@ -1,4 +1,9 @@ --- + - name: Create /services + file: + path: /services + state: directory + - name: Clean local checker services cache become: no local_action: @@ -24,7 +29,7 @@ with_dict: "{{ vulnerable_services }}" - name: pull/build service - shell: docker-compose pull && docker-compose build + shell: docker compose pull && docker compose build args: chdir: /services/checker_{{ item.key }} with_dict: "{{ vulnerable_services }}" diff --git a/ansible/roles/vuln_services/tasks/main.yml b/ansible/roles/vuln_services/tasks/main.yml index f29777b..8d808b7 100644 --- a/ansible/roles/vuln_services/tasks/main.yml +++ b/ansible/roles/vuln_services/tasks/main.yml @@ -1,4 +1,9 @@ --- + - name: Create /services + file: + path: /services + state: directory + - name: Clean local services cache become: no local_action: @@ -23,7 +28,7 @@ with_dict: "{{ vulnerable_services }}" - name: pull/build service - shell: docker-compose pull && docker-compose build + shell: docker compose pull && docker compose build args: chdir: /services/{{ item.key }} with_dict: "{{ vulnerable_services }}" diff --git a/terraform/.terraform.tfstate.lock.info b/terraform/.terraform.tfstate.lock.info deleted file mode 100644 index e3a9d48..0000000 --- a/terraform/.terraform.tfstate.lock.info +++ /dev/null @@ -1 +0,0 @@ -{"ID":"2da0948e-8d55-f4f6-b253-f9d2c1498ba9","Operation":"OperationTypeApply","Info":"","Who":"root@d9db1bc35b27","Version":"1.0.11","Created":"2023-11-16T21:08:30.177346053Z","Path":"terraform.tfstate"} \ No newline at end of file diff --git a/terraform/bambictf.tf b/terraform/bambictf.tf index 93734b6..17e6ed7 100644 --- a/terraform/bambictf.tf +++ b/terraform/bambictf.tf @@ -12,21 +12,29 @@ terraform { required_version = ">= 1.0" } -variable "HETZNERDNS_TOKEN" {} -variable "hetznerdns_zone" {} +variable "HCLOUD_TOKEN" { + type = string + nullable = false + sensitive = true +} +variable "HETZNERDNS_TOKEN" { + type = string + sensitive = true +} +variable "hetznerdns_zone" { + type = string +} variable "hetznerdns_suffix" {} -variable "HCLOUD_TOKEN" {} - variable "router_count" {} variable "checker_count" {} -variable "vulnbox_count" {} - -variable "router_type" {} +variable "router_type" { + type = string + default = "cpx31" + nullable = false +} variable "checker_type" {} variable "engine_type" {} variable "elk_type" {} -variable "vulnbox_type" {} - variable "home_location" {} provider "hcloud" { @@ -63,12 +71,6 @@ data "hcloud_image" "bambichecker" { most_recent = true } -data "hcloud_image" "bambivulnbox" { - with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null - name = var.vulnbox_count > 0 ? null : "debian-10" - most_recent = true -} - data "hcloud_ssh_keys" "all_keys" { with_selector = "type=admin" } @@ -211,29 +213,3 @@ resource "hcloud_server" "bambielk" { ) } -resource "hcloud_server" "bambivulnbox" { - name = "vulnbox${count.index + 1}" - image = data.hcloud_image.bambivulnbox.id - location = var.home_location - server_type = var.vulnbox_type - count = var.vulnbox_count - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_vulnbox.tftpl", { - wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), - index = count.index, - id = "${count.index + 1}", - router_ips = hcloud_floating_ip.bambirouter_ip - } - ) -} - -resource "hetznerdns_record" "bambivulnbox_dns" { - count = var.vulnbox_count - zone_id = data.hetznerdns_zone.zone.id - name = "team${count.index + 1}${var.hetznerdns_suffix}" - value = hcloud_server.bambivulnbox[count.index].ipv4_address - type = "A" - ttl = 60 -} diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf new file mode 100644 index 0000000..93cbac5 --- /dev/null +++ b/terraform/bambivulnbox.tf @@ -0,0 +1,36 @@ +variable "vulnbox_type" {} +variable "vulnbox_count" {} + +data "hcloud_image" "bambivulnbox" { + with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null + name = var.vulnbox_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hetznerdns_record" "bambivulnbox_dns" { + count = var.vulnbox_count + zone_id = data.hetznerdns_zone.zone.id + name = "team${count.index + 1}${var.hetznerdns_suffix}" + value = hcloud_server.bambivulnbox[count.index].ipv4_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambivulnbox" { + name = "vulnbox${count.index + 1}" + image = data.hcloud_image.bambivulnbox.id + location = var.home_location + server_type = var.vulnbox_type + count = var.vulnbox_count + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_vulnbox.tftpl", { + wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), + index = count.index, + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip + } + ) +} + From 121f9d793062f8af4ed5602d82d559cc2ab18718 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 17 Nov 2023 21:52:06 +0100 Subject: [PATCH 08/61] restructure into separate files --- Dockerfile | 2 +- terraform/bambichecker.tf | 44 ++++++ terraform/bambictf.tf | 215 ------------------------------ terraform/bambielk.tf | 57 ++++++++ terraform/bambiengine.tf | 57 ++++++++ terraform/bambirouter.tf | 58 ++++++++ terraform/bambivulnbox.tf | 80 ++++++----- terraform/main.tf | 59 ++++++++ terraform/user_data_checker.tftpl | 4 +- terraform/user_data_elk.tftpl | 4 +- terraform/user_data_engine.tftpl | 4 +- terraform/user_data_router.tftpl | 4 +- 12 files changed, 328 insertions(+), 260 deletions(-) create mode 100644 terraform/bambichecker.tf delete mode 100644 terraform/bambictf.tf create mode 100644 terraform/bambielk.tf create mode 100644 terraform/bambiengine.tf create mode 100644 terraform/bambirouter.tf create mode 100644 terraform/main.tf diff --git a/Dockerfile b/Dockerfile index cabe44e..d4f255d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN add-apt-repository --yes --update ppa:ansible/ansible && apt-get install -y # Terrorform RUN ls -alh /usr/local/bin -RUN curl https://releases.hashicorp.com/terraform/1.0.11/terraform_1.0.11_linux_amd64.zip > terraform.zip && \ +RUN curl https://releases.hashicorp.com/terraform/1.6.4/terraform_1.6.4_linux_amd64.zip > terraform.zip && \ unzip terraform.zip && \ mv terraform /usr/local/bin/ diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf new file mode 100644 index 0000000..fd0e63a --- /dev/null +++ b/terraform/bambichecker.tf @@ -0,0 +1,44 @@ +variable "checker_type" { + type = string + default = "cpx31" + nullable = false +} + +variable "checker_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambichecker" { + with_selector = var.checker_count > 0 ? "type=bambichecker" : null + name = var.checker_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hetznerdns_record" "bambchecker_dns" { + count = var.checker_count + zone_id = data.hetznerdns_zone.zone[0].id + name = "checker${count.index + 1}${local.subdomain}" + value = hcloud_server.bambichecker[count.index].ipv4_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambichecker" { + count = var.checker_count + name = "bambichecker" + image = data.hcloud_image.bambichecker.id + location = var.home_location + server_type = var.checker_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_checker.tftpl", { + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", + engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + } + ) +} diff --git a/terraform/bambictf.tf b/terraform/bambictf.tf deleted file mode 100644 index 17e6ed7..0000000 --- a/terraform/bambictf.tf +++ /dev/null @@ -1,215 +0,0 @@ -terraform { - required_providers { - hetznerdns = { - source = "timohirt/hetznerdns" - version = "2.2.0" - } - hcloud = { - source = "hetznercloud/hcloud" - version = "1.35.2" - } - } - required_version = ">= 1.0" -} - -variable "HCLOUD_TOKEN" { - type = string - nullable = false - sensitive = true -} -variable "HETZNERDNS_TOKEN" { - type = string - sensitive = true -} -variable "hetznerdns_zone" { - type = string -} -variable "hetznerdns_suffix" {} -variable "router_count" {} -variable "checker_count" {} -variable "router_type" { - type = string - default = "cpx31" - nullable = false -} -variable "checker_type" {} -variable "engine_type" {} -variable "elk_type" {} -variable "home_location" {} - -provider "hcloud" { - token = var.HCLOUD_TOKEN -} - -provider "hetznerdns" { - apitoken = var.HETZNERDNS_TOKEN -} - -data "hetznerdns_zone" "zone" { - name = var.hetznerdns_zone -} - -data "hcloud_image" "bambirouter" { - with_selector = var.router_count > 0 ? "type=bambirouter" : null - name = var.router_count > 0 ? null : "debian-10" - most_recent = true -} - -data "hcloud_image" "bambiengine" { - with_selector = "type=bambiengine" - most_recent = true -} - -data "hcloud_image" "bambielk" { - with_selector = "type=bambielk" - most_recent = true -} - -data "hcloud_image" "bambichecker" { - with_selector = var.checker_count > 0 ? "type=bambichecker" : null - name = var.checker_count > 0 ? null : "debian-10" - most_recent = true -} - -data "hcloud_ssh_keys" "all_keys" { - with_selector = "type=admin" -} - -resource "hcloud_floating_ip" "bambirouter_ip" { - count = var.router_count - type = "ipv4" - name = "router${count.index + 1}" - home_location = var.home_location -} - -resource "hcloud_floating_ip_assignment" "bambirouter_ipa" { - count = var.router_count - floating_ip_id = hcloud_floating_ip.bambirouter_ip[count.index].id - server_id = hcloud_server.bambirouter[count.index].id -} - -resource "hetznerdns_record" "bambirouter_dns" { - count = var.router_count - zone_id = data.hetznerdns_zone.zone.id - name = "router${count.index + 1}${var.hetznerdns_suffix}" - value = hcloud_floating_ip.bambirouter_ip[count.index].ip_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambirouter" { - count = var.router_count - name = "router${count.index + 1}" - image = data.hcloud_image.bambirouter.id - location = var.home_location - server_type = var.router_type - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_router.tftpl", { - index = count.index, - id = "${count.index + 1}", - router_ips = hcloud_floating_ip.bambirouter_ip, - elk = hcloud_floating_ip.bambielk_ip, - engine = hcloud_floating_ip.bambiengine_ip, - } - ) -} - -resource "hcloud_floating_ip" "bambiengine_ip" { - type = "ipv4" - name = "engine" - home_location = var.home_location -} - -resource "hcloud_floating_ip_assignment" "bambiengine_ipa" { - floating_ip_id = hcloud_floating_ip.bambiengine_ip.id - server_id = hcloud_server.bambiengine.id -} - -resource "hetznerdns_record" "bambiengine_dns" { - zone_id = data.hetznerdns_zone.zone.id - name = "engine${var.hetznerdns_suffix}" - value = hcloud_floating_ip.bambiengine_ip.ip_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambiengine" { - name = "bambiengine" - image = data.hcloud_image.bambiengine.id - location = var.home_location - server_type = var.engine_type - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_engine.tftpl", { - router_ips = hcloud_floating_ip.bambirouter_ip, - elk = hcloud_floating_ip.bambielk_ip, - engine = hcloud_floating_ip.bambiengine_ip, - } - ) -} - -resource "hetznerdns_record" "bambchecker_dns" { - count = var.checker_count - zone_id = data.hetznerdns_zone.zone.id - name = "checker${count.index + 1}${var.hetznerdns_suffix}" - value = hcloud_server.bambichecker[count.index].ipv4_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambichecker" { - name = "bambichecker" - image = data.hcloud_image.bambichecker.id - location = var.home_location - server_type = var.checker_type - count = var.checker_count - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_checker.tftpl", { - id = "${count.index + 1}", - router_ips = hcloud_floating_ip.bambirouter_ip, - elk = hcloud_floating_ip.bambielk_ip, - engine = hcloud_floating_ip.bambiengine_ip, - } - ) -} - -resource "hcloud_floating_ip" "bambielk_ip" { - type = "ipv4" - name = "elk" - home_location = var.home_location -} - -resource "hcloud_floating_ip_assignment" "bambielk_ipa" { - floating_ip_id = hcloud_floating_ip.bambielk_ip.id - server_id = hcloud_server.bambielk.id -} - -resource "hetznerdns_record" "bambielk_dns" { - zone_id = data.hetznerdns_zone.zone.id - name = "elk${var.hetznerdns_suffix}" - value = hcloud_server.bambielk.ipv4_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambielk" { - name = "bambielk" - image = data.hcloud_image.bambielk.id - location = var.home_location - server_type = var.elk_type - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_elk.tftpl", { - router_ips = hcloud_floating_ip.bambirouter_ip, - elk = hcloud_floating_ip.bambielk_ip, - engine = hcloud_floating_ip.bambiengine_ip, - } - ) -} - diff --git a/terraform/bambielk.tf b/terraform/bambielk.tf new file mode 100644 index 0000000..8875950 --- /dev/null +++ b/terraform/bambielk.tf @@ -0,0 +1,57 @@ +variable "elk_type" { + type = string + default = "cpx31" + nullable = false +} + +variable "elk_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambielk" { + count = var.elk_count > 0 ? 1 : 0 + with_selector = var.elk_count > 0 ? "type=bambielk" : null + name = var.elk_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hcloud_floating_ip" "bambielk_ip" { + count = var.elk_count + type = "ipv4" + name = "elk" + home_location = var.home_location +} + +resource "hcloud_floating_ip_assignment" "bambielk_ipa" { + count = var.elk_count + floating_ip_id = hcloud_floating_ip.bambielk_ip[0].id + server_id = hcloud_server.bambielk[0].id +} + +resource "hetznerdns_record" "bambielk_dns" { + count = var.elk_count + zone_id = data.hetznerdns_zone.zone[0].id + name = "elk${local.subdomain}" + value = hcloud_server.bambielk[0].ipv4_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambielk" { + count = var.elk_count + name = "bambielk" + image = data.hcloud_image.bambielk[0].id + location = var.home_location + server_type = var.elk_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_elk.tftpl", { + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", + engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + } + ) +} diff --git a/terraform/bambiengine.tf b/terraform/bambiengine.tf new file mode 100644 index 0000000..08deb82 --- /dev/null +++ b/terraform/bambiengine.tf @@ -0,0 +1,57 @@ +variable "engine_type" { + type = string + default = "cpx31" + nullable = false +} + +variable "engine_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambiengine" { + count = var.engine_count > 0 ? 1 : 0 + with_selector = var.engine_count > 0 ? "type=bambiengine" : null + name = var.engine_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hcloud_floating_ip" "bambiengine_ip" { + count = var.engine_count + type = "ipv4" + name = "engine" + home_location = var.home_location +} + +resource "hcloud_floating_ip_assignment" "bambiengine_ipa" { + count = var.engine_count + floating_ip_id = hcloud_floating_ip.bambiengine_ip[0].id + server_id = hcloud_server.bambiengine[0].id +} + +resource "hetznerdns_record" "bambiengine_dns" { + count = var.engine_count + zone_id = data.hetznerdns_zone.zone[0].id + name = "engine${local.subdomain}" + value = hcloud_floating_ip.bambiengine_ip[0].ip_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambiengine" { + count = var.engine_count + name = "bambiengine" + image = data.hcloud_image.bambiengine[0].id + location = var.home_location + server_type = var.engine_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_engine.tftpl", { + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", + engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + } + ) +} diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf new file mode 100644 index 0000000..19bea6b --- /dev/null +++ b/terraform/bambirouter.tf @@ -0,0 +1,58 @@ +variable "router_type" { + type = string + default = "cpx31" + nullable = false +} + +variable "router_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambirouter" { + with_selector = var.router_count > 0 ? "type=bambirouter" : null + name = var.router_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hcloud_floating_ip" "bambirouter_ip" { + count = var.router_count + type = "ipv4" + name = "router${count.index + 1}" + home_location = var.home_location +} + +resource "hcloud_floating_ip_assignment" "bambirouter_ipa" { + count = var.router_count + floating_ip_id = hcloud_floating_ip.bambirouter_ip[count.index].id + server_id = hcloud_server.bambirouter[count.index].id +} + +resource "hetznerdns_record" "bambirouter_dns" { + count = var.router_count + zone_id = data.hetznerdns_zone.zone[0].id + name = "router${count.index + 1}${local.subdomain}" + value = hcloud_floating_ip.bambirouter_ip[count.index].ip_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambirouter" { + count = var.router_count + name = "router${count.index + 1}" + image = data.hcloud_image.bambirouter.id + location = var.home_location + server_type = var.router_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_router.tftpl", { + index = count.index, + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", + engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + } + ) +} diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf index 93cbac5..407f6b8 100644 --- a/terraform/bambivulnbox.tf +++ b/terraform/bambivulnbox.tf @@ -1,36 +1,44 @@ -variable "vulnbox_type" {} -variable "vulnbox_count" {} - -data "hcloud_image" "bambivulnbox" { - with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null - name = var.vulnbox_count > 0 ? null : "debian-10" - most_recent = true -} - -resource "hetznerdns_record" "bambivulnbox_dns" { - count = var.vulnbox_count - zone_id = data.hetznerdns_zone.zone.id - name = "team${count.index + 1}${var.hetznerdns_suffix}" - value = hcloud_server.bambivulnbox[count.index].ipv4_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambivulnbox" { - name = "vulnbox${count.index + 1}" - image = data.hcloud_image.bambivulnbox.id - location = var.home_location - server_type = var.vulnbox_type - count = var.vulnbox_count - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_vulnbox.tftpl", { - wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), - index = count.index, - id = "${count.index + 1}", - router_ips = hcloud_floating_ip.bambirouter_ip - } - ) -} - +variable "vulnbox_type" { + type = string + default = "cpx21" + nullable = false +} + +variable "vulnbox_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambivulnbox" { + with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null + name = var.vulnbox_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hetznerdns_record" "bambivulnbox_dns" { + count = var.hetznerdns_zone != null ? var.vulnbox_count : 0 + zone_id = data.hetznerdns_zone.zone[0].id + name = "team${count.index + 1}${local.subdomain}" + value = hcloud_server.bambivulnbox[count.index].ipv4_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambivulnbox" { + name = "vulnbox${count.index + 1}" + image = data.hcloud_image.bambivulnbox.id + location = var.home_location + server_type = var.vulnbox_type + count = var.vulnbox_count + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_vulnbox.tftpl", { + wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), + index = count.index, + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip + } + ) +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..0e55d57 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,59 @@ +terraform { + required_providers { + hetznerdns = { + source = "timohirt/hetznerdns" + version = "2.2.0" + } + hcloud = { + source = "hetznercloud/hcloud" + version = "1.35.2" + } + } + required_version = ">= 1.0" +} + +variable "HCLOUD_TOKEN" { + type = string + nullable = false + sensitive = true +} +variable "HETZNERDNS_TOKEN" { + type = string + sensitive = true +} + +variable "home_location" { + type = string + nullable = false + default = "fsn1" +} + +variable "hetznerdns_zone" { + type = string + default = null +} +variable "subdomain" { + type = string + default = "" +} + +locals { + subdomain = var.subdomain != null ? ".${var.subdomain}" : "" +} + +provider "hcloud" { + token = var.HCLOUD_TOKEN +} + +provider "hetznerdns" { + apitoken = var.HETZNERDNS_TOKEN +} + +data "hetznerdns_zone" "zone" { + count = var.hetznerdns_zone != null ? 1 : 0 + name = var.hetznerdns_zone +} + +data "hcloud_ssh_keys" "all_keys" { + with_selector = "type=admin" +} diff --git a/terraform/user_data_checker.tftpl b/terraform/user_data_checker.tftpl index 90c065c..ca8a818 100644 --- a/terraform/user_data_checker.tftpl +++ b/terraform/user_data_checker.tftpl @@ -3,8 +3,8 @@ set -e # Network mv "/etc/wireguard/checker${id}.conf" "/etc/wireguard/internal.conf" -sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine.ip_address}#g" "/etc/wireguard/internal.conf" -sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk.ip_address}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" %{ for router_index, router in router_ips ~} sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" diff --git a/terraform/user_data_elk.tftpl b/terraform/user_data_elk.tftpl index fbf210f..30a4964 100644 --- a/terraform/user_data_elk.tftpl +++ b/terraform/user_data_elk.tftpl @@ -8,11 +8,11 @@ network: ethernets: eth0: addresses: - - ${elk.ip_address}/32 + - ${elk}/32 EOF netplan apply -sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine.ip_address}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" %{ for router_index, router in router_ips ~} sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" diff --git a/terraform/user_data_engine.tftpl b/terraform/user_data_engine.tftpl index 4bc686f..561840d 100644 --- a/terraform/user_data_engine.tftpl +++ b/terraform/user_data_engine.tftpl @@ -7,7 +7,7 @@ network: ethernets: eth0: addresses: - - ${engine.ip_address}/32 + - ${engine}/32 EOF netplan apply @@ -15,7 +15,7 @@ netplan apply sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" %{ endfor ~} -sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk.ip_address}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" systemctl enable "wg-quick@internal" systemctl start "wg-quick@internal" diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 7c95430..d5cb937 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -23,8 +23,8 @@ netplan apply mv "/etc/wireguard/router${id}_internal.conf" "/etc/wireguard/internal.conf" mv "/etc/wireguard/router${id}_game.conf" "/etc/wireguard/router.conf" -sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine.ip_address}#g" "/etc/wireguard/internal.conf" -sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk.ip_address}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" systemctl enable "wg-quick@internal" systemctl start "wg-quick@internal" From 301a22df20c849e19045eae11b086b4d5a23fd62 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 18 Nov 2023 10:47:32 +0100 Subject: [PATCH 09/61] make dns optional (finally!) --- terraform/bambichecker.tf | 2 +- terraform/bambielk.tf | 6 +++++- terraform/bambiengine.tf | 6 +++++- terraform/bambirouter.tf | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index fd0e63a..7bdd397 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -17,7 +17,7 @@ data "hcloud_image" "bambichecker" { } resource "hetznerdns_record" "bambchecker_dns" { - count = var.checker_count + count = var.hetznerdns_zone != null ? var.checker_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "checker${count.index + 1}${local.subdomain}" value = hcloud_server.bambichecker[count.index].ipv4_address diff --git a/terraform/bambielk.tf b/terraform/bambielk.tf index 8875950..2dd8ea6 100644 --- a/terraform/bambielk.tf +++ b/terraform/bambielk.tf @@ -8,6 +8,10 @@ variable "elk_count" { type = number default = 0 nullable = false + validation { + condition = var.elk_count == 0 || var.elk_count == 1 + error_message = "elk_count must be 1 or 0" + } } data "hcloud_image" "bambielk" { @@ -31,7 +35,7 @@ resource "hcloud_floating_ip_assignment" "bambielk_ipa" { } resource "hetznerdns_record" "bambielk_dns" { - count = var.elk_count + count = var.hetznerdns_zone != null ? var.elk_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "elk${local.subdomain}" value = hcloud_server.bambielk[0].ipv4_address diff --git a/terraform/bambiengine.tf b/terraform/bambiengine.tf index 08deb82..de012cf 100644 --- a/terraform/bambiengine.tf +++ b/terraform/bambiengine.tf @@ -8,6 +8,10 @@ variable "engine_count" { type = number default = 0 nullable = false + validation { + condition = var.engine_count == 0 || var.engine_count == 1 + error_message = "engine_count must be 1 or 0" + } } data "hcloud_image" "bambiengine" { @@ -31,7 +35,7 @@ resource "hcloud_floating_ip_assignment" "bambiengine_ipa" { } resource "hetznerdns_record" "bambiengine_dns" { - count = var.engine_count + count = var.hetznerdns_zone != null ? var.engine_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "engine${local.subdomain}" value = hcloud_floating_ip.bambiengine_ip[0].ip_address diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf index 19bea6b..6d575e5 100644 --- a/terraform/bambirouter.tf +++ b/terraform/bambirouter.tf @@ -30,7 +30,7 @@ resource "hcloud_floating_ip_assignment" "bambirouter_ipa" { } resource "hetznerdns_record" "bambirouter_dns" { - count = var.router_count + count = var.hetznerdns_zone != null ? var.router_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "router${count.index + 1}${local.subdomain}" value = hcloud_floating_ip.bambirouter_ip[count.index].ip_address From 6e04429eb5813de9355641728d5c5dd37a6bda1e Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 18 Nov 2023 21:00:45 +0100 Subject: [PATCH 10/61] yarp this looks good --- terraform/{main.tf => bambi.tf} | 2 ++ terraform/bambichecker.tf | 2 +- terraform/bambiengine.tf | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) rename terraform/{main.tf => bambi.tf} (99%) diff --git a/terraform/main.tf b/terraform/bambi.tf similarity index 99% rename from terraform/main.tf rename to terraform/bambi.tf index 0e55d57..4dc10e4 100644 --- a/terraform/main.tf +++ b/terraform/bambi.tf @@ -17,6 +17,7 @@ variable "HCLOUD_TOKEN" { nullable = false sensitive = true } + variable "HETZNERDNS_TOKEN" { type = string sensitive = true @@ -32,6 +33,7 @@ variable "hetznerdns_zone" { type = string default = null } + variable "subdomain" { type = string default = "" diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index 7bdd397..aba77f7 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -27,7 +27,7 @@ resource "hetznerdns_record" "bambchecker_dns" { resource "hcloud_server" "bambichecker" { count = var.checker_count - name = "bambichecker" + name = "checker${count.index + 1}" image = data.hcloud_image.bambichecker.id location = var.home_location server_type = var.checker_type diff --git a/terraform/bambiengine.tf b/terraform/bambiengine.tf index de012cf..19baa2d 100644 --- a/terraform/bambiengine.tf +++ b/terraform/bambiengine.tf @@ -45,7 +45,7 @@ resource "hetznerdns_record" "bambiengine_dns" { resource "hcloud_server" "bambiengine" { count = var.engine_count - name = "bambiengine" + name = "engine" image = data.hcloud_image.bambiengine[0].id location = var.home_location server_type = var.engine_type From 24acc255c1e0b5754933eb15ca5066516e0cb458 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 18 Nov 2023 23:00:06 +0100 Subject: [PATCH 11/61] cleanup configgen --- configgen/configgen/__init__.py | 2 +- configgen/configgen/gen_wireguard_game.py | 39 ++++------------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index 41faa06..645458b 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -23,7 +23,7 @@ def main() -> None: dns: str = args.dns routers: int = args.routers checkers: int = args.checkers - logger.info(f"Generating for {teams} teams, {routers} gws and {checkers} checkers") + logger.info(f"Generating for {teams} teams, {routers} routers and {checkers} checkers") prepare_directories(teams) gen_wireguard_game(teams, dns, routers) gen_wireguard_internal(teams, checkers, routers) diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index c37c540..61f708d 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Optional, Tuple +from typing import Optional from configgen.util import ( CIDR_GAME, @@ -13,10 +13,9 @@ TeamConfig, WireguardRouterConfig, create_config_file, - gen_wg_priv, - gen_wg_pub, get_router_cidr_game, get_vulnbox_cidr, + get_wg, ) logger = logging.getLogger(__file__) @@ -30,10 +29,10 @@ def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: team_portal_configs: list[TeamConfig] = [] team_terraform_configs: list[TeamConfig] = [] - # Generate gw configs + # Generate router configs for router_id in range(1, routers + 1): local_ip = get_router_cidr_game(router_id) - private_key, public_key = get_router_wg(router_id) + private_key, public_key = get_wg(Path(f"wg_game/router{router_id}.key")) router_configs.append( WireguardRouterConfig( router_id=router_id, @@ -46,9 +45,9 @@ def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: ) ) - # Generate team configs and add the peers to gw configs + # Generate team configs and add the peers to router configs for team in range(1, teams + 1): - private_key, public_key = get_team_wg(team) + private_key, public_key = get_wg(Path(f"wg_game/team{team}.key")) x, y = (team // 250), (team % 250) router_index = (team - 1) % routers router_config = router_configs[router_index] @@ -104,32 +103,6 @@ def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: ).write_text(create_config_file(router_config)) -def get_team_wg(team: int) -> Tuple[str, str]: - try: - priv = Path(f"{DATA_DIR}/wg_game/team{team}.key").read_text() - return priv, gen_wg_pub(priv) - except FileNotFoundError: - pass - - logger.debug(f"Generating fresh privkey for team{team}") - priv = gen_wg_priv() - Path(f"{DATA_DIR}/wg_game/team{team}.key").write_text(priv) - return priv, gen_wg_pub(priv) - - -def get_router_wg(gw: int) -> Tuple[str, str]: - try: - priv = Path(f"{DATA_DIR}/wg_game/router{gw}.key").read_text() - return priv, gen_wg_pub(priv) - except FileNotFoundError: - pass - - logger.debug(f"Generating fresh privkey for gw{gw}") - priv = gen_wg_priv() - Path(f"{DATA_DIR}/wg_game/router{gw}.key").write_text(priv) - return priv, gen_wg_pub(priv) - - def gen_team_config( private_key: str, public_key: str, From 2bfeb8344b757c548935850609d8b479a0e9a517 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 18 Nov 2023 23:06:06 +0100 Subject: [PATCH 12/61] make dns optional in configgen --- configgen/configgen/__init__.py | 8 +++++--- configgen/configgen/gen_wireguard_game.py | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index 645458b..bc84492 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -3,6 +3,7 @@ import secrets import shutil from pathlib import Path +from typing import Optional from configgen.gen_wireguard_game import gen_wireguard_game from configgen.gen_wireguard_internal import gen_wireguard_internal @@ -15,12 +16,12 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--teams", type=int, required=True) - parser.add_argument("--dns", type=str, required=True) + parser.add_argument("--dns", type=str, default=None) parser.add_argument("--routers", type=int, default=2) parser.add_argument("--checkers", type=int, default=10) args = parser.parse_args() teams: int = args.teams - dns: str = args.dns + dns: Optional[str] = args.dns routers: int = args.routers checkers: int = args.checkers logger.info(f"Generating for {teams} teams, {routers} routers and {checkers} checkers") @@ -28,7 +29,8 @@ def main() -> None: gen_wireguard_game(teams, dns, routers) gen_wireguard_internal(teams, checkers, routers) gen_passwords(teams) - gen_userdata_portal(teams) + if dns: + gen_userdata_portal(teams) def prepare_directories(teams: int) -> None: diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index 61f708d..aca0c50 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__file__) -def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: +def gen_wireguard_game(teams: int, dns: Optional[str], routers: int) -> None: """ Generates the game wireguard config files as needed, reusing keys (if present) """ @@ -51,9 +51,10 @@ def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: x, y = (team // 250), (team % 250) router_index = (team - 1) % routers router_config = router_configs[router_index] - team_portal_configs.append( - gen_team_config(private_key, public_key, team, router_config, dns) - ) + if dns != None: + team_portal_configs.append( + gen_team_config(private_key, public_key, team, router_config, dns) + ) team_terraform_configs.append( gen_team_config(private_key, public_key, team, router_config, None) ) @@ -87,10 +88,11 @@ def gen_wireguard_game(teams: int, dns: str, routers: int) -> None: ) ) - for team_portal_config in team_portal_configs: - Path( - f"{DATA_DIR}/export/portal/team{team_portal_config.team_id}/game.conf" - ).write_text(create_config_file(team_portal_config)) + if dns != None: + for team_portal_config in team_portal_configs: + Path( + f"{DATA_DIR}/export/portal/team{team_portal_config.team_id}/game.conf" + ).write_text(create_config_file(team_portal_config)) for team_terraform_config in team_terraform_configs: Path( From c155fe538d32ba4ec3797b243debe2206fc70c9a Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 19 Nov 2023 19:05:54 +0100 Subject: [PATCH 13/61] begin arkime impl --- ansible/bambiarkime.yml | 26 +++ ansible/roles/bambi-arkime/files/Dockerfile | 11 + .../bambi-arkime/files/arkime-capture.sh | 7 + .../roles/bambi-arkime/files/arkime-viewer.sh | 7 + ansible/roles/bambi-arkime/files/config.ini | 214 ++++++++++++++++++ .../files/docker-compose.arkime.yml | 11 + .../files/docker-compose.elasticsearch.yml | 16 ++ ansible/roles/bambi-arkime/files/init.sh | 16 ++ ansible/roles/bambi-arkime/meta/main.yml | 3 + ansible/roles/bambi-arkime/tasks/main.yml | 56 +++++ ansible/roles/wireguard_arkime/tasks/main.yml | 5 + configgen/configgen/__init__.py | 5 +- configgen/configgen/gen_wireguard_game.py | 10 +- configgen/configgen/gen_wireguard_internal.py | 38 +++- configgen/configgen/util.py | 11 +- packer/bambiarkime.json | 26 +++ terraform/bambiarkime.tf | 44 ++++ terraform/bambichecker.tf | 88 +++---- terraform/terraform.tfvars.sample | 5 +- terraform/user_data_arkime.tftpl | 19 ++ 20 files changed, 560 insertions(+), 58 deletions(-) create mode 100644 ansible/bambiarkime.yml create mode 100644 ansible/roles/bambi-arkime/files/Dockerfile create mode 100644 ansible/roles/bambi-arkime/files/arkime-capture.sh create mode 100644 ansible/roles/bambi-arkime/files/arkime-viewer.sh create mode 100644 ansible/roles/bambi-arkime/files/config.ini create mode 100644 ansible/roles/bambi-arkime/files/docker-compose.arkime.yml create mode 100644 ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml create mode 100644 ansible/roles/bambi-arkime/files/init.sh create mode 100644 ansible/roles/bambi-arkime/meta/main.yml create mode 100644 ansible/roles/bambi-arkime/tasks/main.yml create mode 100644 ansible/roles/wireguard_arkime/tasks/main.yml create mode 100644 packer/bambiarkime.json create mode 100644 terraform/bambiarkime.tf create mode 100644 terraform/user_data_arkime.tftpl diff --git a/ansible/bambiarkime.yml b/ansible/bambiarkime.yml new file mode 100644 index 0000000..ff02487 --- /dev/null +++ b/ansible/bambiarkime.yml @@ -0,0 +1,26 @@ +--- + - hosts: all + become: yes + become_method: sudo + vars_files: + - config_bambi.yml + + roles: + - bambi-ssh-keys + - firewall + - bambi-arkime + - wireguard + - wireguard_arkime + - docker + - role: journalbeat + vars: + elk: 192.168.3.0 + - role: metricbeat + vars: + elk: 192.168.3.0 + - docker-block-external + - role: programs + vars: + program_list: + - "tmux" + - "git" diff --git a/ansible/roles/bambi-arkime/files/Dockerfile b/ansible/roles/bambi-arkime/files/Dockerfile new file mode 100644 index 0000000..7ea2a3b --- /dev/null +++ b/ansible/roles/bambi-arkime/files/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y libwww-perl libjson-perl ethtool libyaml-dev liblua5.4-0 libmaxminddb0 libcurl4 libpcap0.8 libglib2.0-0 libnghttp2-14 libyara8 librdkafka1 curl + +WORKDIR /BambiArkime +ADD https://github.com/arkime/arkime/releases/download/v5.0.0-rc1/arkime_5.0.0-rc1-1.ubuntu2204_amd64.deb . +RUN dpkg -i ./*.deb; apt-get install -fy + +RUN touch /opt/arkime/etc/oui.txt +RUN touch /opt/arkime/etc/ipv4-address-space.csv + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/bambi-arkime/files/arkime-capture.sh new file mode 100644 index 0000000..e8a8de3 --- /dev/null +++ b/ansible/roles/bambi-arkime/files/arkime-capture.sh @@ -0,0 +1,7 @@ +cd /opt/arkime +while : +do + echo "Starting Arkime capture" + /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /opt/arkime/raw -m --host $1 + sleep 5 +done diff --git a/ansible/roles/bambi-arkime/files/arkime-viewer.sh b/ansible/roles/bambi-arkime/files/arkime-viewer.sh new file mode 100644 index 0000000..3c14f4e --- /dev/null +++ b/ansible/roles/bambi-arkime/files/arkime-viewer.sh @@ -0,0 +1,7 @@ +cd /opt/arkime/viewer +while : +do + echo "Starting Arkime viewer" + /opt/arkime/bin/node viewer.js -c /opt/arkime/etc/config.ini --host $1 + sleep 5 +done diff --git a/ansible/roles/bambi-arkime/files/config.ini b/ansible/roles/bambi-arkime/files/config.ini new file mode 100644 index 0000000..0bd56a3 --- /dev/null +++ b/ansible/roles/bambi-arkime/files/config.ini @@ -0,0 +1,214 @@ +[default] +authMode=anonymous +elasticsearch=http://[[ARKIME]]:9200 +rotateIndex=daily +# Uncomment if this node should process the cron queries and packet search jobs, only ONE node should +# process cron queries and packet search jobs +cronQueries=auto +passwordSecret=stronkpassword +httpRealm=Moloch + +# Semicolon ';' seperated list of interfaces to listen on for traffic +interface=eth1 +# The directory to save raw pcap files to +pcapDir=/opt/arkime/raw + +# The max raw pcap file size in gigabytes, with a max value of 36G. +# The disk should have room for at least 10*maxFileSizeG +maxFileSizeG=12 + +# The max time in minutes between rotating pcap files. Default is 0, which means +# only rotate based on current file size and the maxFileSizeG variable +#maxFileTimeM=60 + +# TCP timeout value. Arkime writes a session record after this many seconds +# of inactivity. +tcpTimeout=600 + +# Arkime writes a session record after this many seconds, no matter if +# active or inactive +tcpSaveTimeout=720 + +# UDP timeout value. Arkime assumes the UDP session is ended after this +# many seconds of inactivity. +udpTimeout=30 + +# ICMP timeout value. Arkime assumes the ICMP session is ended after this +# many seconds of inactivity. +icmpTimeout=10 + +# An aproximiate maximum number of active sessions Arkime will try and monitor +maxStreams=1000000 + +# Arkime writes a session record after this many packets +maxPackets=10000 + +# Delete pcap files when free space is lower then this in gigabytes OR it can be +# expressed as a percentage (ex: 5%). This does NOT delete the session records in +# the database. It is recommended this value is between 5% and 10% of the disk. +# Database deletes are done by the db.pl expire script +freeSpaceG=5% + +# The port to listen on, by default 8005 +#viewPort=8005 + +# The host/ip to listen on, by default 0.0.0.0 which is ALL +#viewHost=localhost + +# A MaxMind account is now required, Arkime checks several install locations, or +# will work without Geo files installed. See https://arkime.com/faq#maxmind +#geoLite2Country=/var/lib/GeoIP/GeoLite2-Country.mmdb;/usr/share/GeoIP/GeoLite2-Country.mmdb;/opt/arkime/etc/GeoLite2-Country.mmdb +#geoLite2ASN=/var/lib/GeoIP/GeoLite2-ASN.mmdb;/usr/share/GeoIP/GeoLite2-ASN.mmdb;/opt/arkime/etc/GeoLite2-ASN.mmdb + +# Path of the rir assignments file +# https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv +rirFile=/opt/arkime/etc/ipv4-address-space.csv + +# Path of the OUI file from whareshark +# https://raw.githubusercontent.com/wireshark/wireshark/release-4.0/manuf +ouiFile=/opt/arkime/etc/oui.txt + +# Arkime rules to allow you specify actions to perform when criteria are met with certain fields or state. +# See https://arkime.com/rulesformat +#rulesFiles=/opt/arkime/etc/arkime.rules + +# User to drop privileges to. The pcapDir must be writable by this user or group below +dropUser=nobody + +# Group to drop privileges to. The pcapDir must be writable by this group or user above +dropGroup=daemon + +# Should we parse extra smtp traffic info +parseSMTP=true + +# Should we parse extra smb traffic info +parseSMB=true + +# Should we parse HTTP QS Values +parseQSValue=false + +# Should we calculate sha256 for bodies +supportSha256=false + +# Only index HTTP request bodies less than this number of bytes */ +maxReqBody=64 + +# Only store request bodies that Utf-8? +reqBodyOnlyUtf8=true + +# Semicolon ';' seperated list of SMTP Headers that have ips, need to have the terminating colon ':' +smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: + +# Semicolon ';' seperated list of directories to load parsers from +parsersDir=/opt/arkime/parsers + +# Semicolon ';' seperated list of directories to load plugins from +pluginsDir=/opt/arkime/plugins + +# Semicolon ';' seperated list of plugins to load and the order to load in +# plugins=tagger.so; netflow.so + +# Plugins to load as root, usually just readers +#rootPlugins=reader-pfring; reader-daq.so + +# Semicolon ';' seperated list of viewer plugins to load and the order to load in +# viewerPlugins=wise.js + +# Specify the max number of indices we calculate spidata for. +# ES will blow up if we allow the spiData to search too many indices. +spiDataMaxIndices=4 + +# Uncomment the following to allow direct uploads. This is experimental +#uploadCommand=/opt/arkime/bin/capture --copy -n {NODE} -r {TMPFILE} -c {CONFIG} {TAGS} + +# Title Template +# _cluster_ = ES cluster name +# _userId_ = logged in User Id +# _userName_ = logged in User Name +# _page_ = internal page name +# _expression_ = current search expression if set, otherwise blank +# _-expression_ = " - " + current search expression if set, otherwise blank, prior spaces removed +# _view_ = current view if set, otherwise blank +# _-view_ = " - " + current view if set, otherwise blank, prior spaces removed +#titleTemplate=_cluster_ - _page_ _-view_ _-expression_ + +# Number of threads processing packets +packetThreads=2 + +# ADVANCED - Semicolon ';' seperated list of files to load for config. Files are loaded +# in order and can replace values set in this file or previous files. +#includes= + +# ADVANCED - How is pcap written to disk +# simple = use O_DIRECT if available, writes in pcapWriteSize chunks, +# a file per packet thread. +# simple-nodirect = don't use O_DIRECT. Required for zfs and others +pcapWriteMethod=simple + +# ADVANCED - Buffer size when writing pcap files. Should be a multiple of the raid 5 or xfs +# stripe size. Defaults to 256k +pcapWriteSize=262143 + +# ADVANCED - Max number of connections to OpenSearch/Elasticsearch +maxESConns=30 +maxESRequests=500 + +logEveryXPackets=100000 +logUnknownProtocols=false +logESRequests=true +logFileCreation=true + + +### High Performance settings +# https://arkime.com/settings#high-performance-settings +# magicMode=basic +# pcapReadMethod=tpacketv3 +# tpacketv3NumThreads=2 +# pcapWriteMethod=simple +# pcapWriteSize=2560000 +# packetThreads=5 +# maxPacketsInQueue=200000 + +### Low Bandwidth settings +# packetThreads=1 +# pcapWriteSize=65536 + +############################################################################## +# override-ips is a special section that overrides the MaxMind databases for +# the fields set, but fields not set will still use MaxMind (example if you set +# tags but not country it will use MaxMind for the country) +# Spaces and capitalization is very important. +# IP Can be a single IP or a CIDR +# Up to 10 tags can be added +# +# ip=tag:TAGNAME1;tag:TAGNAME2;country:3LetterUpperCaseCountry;asn:ASN STRING +#[override-ips] +#10.1.0.0/16=tag:ny-office;country:USA;asn:AS0000 This is an ASN + +############################################################################## +# It is possible to define in the config file extra http/email headers +# to index. They are accessed using the expression http.<fieldname> and +# email.<fieldname> with optional .cnt expressions +# +# Possible config atributes for all headers +# type:<string> (string|integer|ip) = data type (default string) +# count:<boolean> = index count of items (default false) +# unique:<boolean> = only record unique items (default true) + +# headers-http-request is used to configure request headers to index +[headers-http-request] +referer=type:string;count:true;unique:true +authorization=type:string;count:true +content-type=type:string;count:true +origin=type:string + +# headers-http-response is used to configure http response headers to index +[headers-http-response] +location=type:string +server=type:string +content-type=type:string;count:true + +# headers-email is used to configure email headers to index +[headers-email] +x-priority=type:integer +authorization=type:string diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml new file mode 100644 index 0000000..46bc932 --- /dev/null +++ b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml @@ -0,0 +1,11 @@ +services: + arkime: + #image: ghcr.io/enoflag/enoarkime:nightly + build: . + restart: unless-stopped + ports: + - 80:8005 + volumes: + - "/pcaps:/opt/arkime/raw" + - ".:/BambiArkime:ro" + - "./config.ini:/opt/arkime/etc/config.ini:ro" diff --git a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml new file mode 100644 index 0000000..f7221d7 --- /dev/null +++ b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml @@ -0,0 +1,16 @@ +services: + elasticsearch: + restart: unless-stopped + image: elasticsearch:8.11.1 + environment: + - xpack.security.enabled=false + - node.name=[[ARKIME]] + - cluster.name=bambiarkime + - cluster.initial_master_nodes=[[INITIAL_MASTER_NODES]] + - discovery.seed_hosts=[[SEED_HOSTS]] + - network.publish_host=[[ARKIME]] + - http.publish_host=[[ARKIME]] + - transport.publish_host=[[ARKIME]] + ports: + - "9200:9200" + - "9300:9300" diff --git a/ansible/roles/bambi-arkime/files/init.sh b/ansible/roles/bambi-arkime/files/init.sh new file mode 100644 index 0000000..8eb45d0 --- /dev/null +++ b/ansible/roles/bambi-arkime/files/init.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -e +elasticsearch="192.168.2.1" + +while ! curl -sq http://$elasticsearch:9200; do + echo "Waiting for elasticsearch to start..."; + sleep 3; +done + +echo "Initializing Arkime Elasticsearch" +/opt/arkime/db/db.pl http://$elasticsearch:9200 init +/opt/arkime/bin/arkime_add_user.sh admin "Admin User" admin --admin + +sh ./arkime-capture.sh bambiarkime1 & +sh ./arkime-viewer.sh bambiarkime1 & +wait diff --git a/ansible/roles/bambi-arkime/meta/main.yml b/ansible/roles/bambi-arkime/meta/main.yml new file mode 100644 index 0000000..1e3630d --- /dev/null +++ b/ansible/roles/bambi-arkime/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: +- role: "docker" +- role: "bambi-ssh-keys" diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml new file mode 100644 index 0000000..310a4ce --- /dev/null +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -0,0 +1,56 @@ +--- + - name: Increase max_map_count + sysctl: + name: vm.max_map_count + value: "262144" + sysctl_set: yes + + - name: Create /services/BambiArkimeElasticsearch + file: + path: /services/BambiArkimeElasticsearch + state: directory + + - name: docker-compose.elasticsearch.yml + copy: + src: docker-compose.elasticsearch.yml + dest: /services/BambiArkimeElasticsearch/docker-compose.yml + + - name: Pull ES + shell: docker compose pull + args: + chdir: /services/BambiArkimeElasticsearch + + - name: Create /services/BambiArkime + file: + path: /services/BambiArkime + state: directory + + - name: Copy docker-compose.arkime.yml + copy: + src: docker-compose.arkime.yml + dest: /services/BambiArkime/docker-compose.yml + + - name: Copy config.ini + copy: + src: config.ini + dest: /services/BambiArkime/config.ini + + - name: Copy Dockerfile + copy: + src: Dockerfile + dest: /services/BambiArkime/Dockerfile + + - name: Copy arkime-capture.sh + copy: + src: arkime-capture.sh + dest: /services/BambiArkime/arkime-capture.sh + + - name: Copy arkime-viewer.sh + copy: + src: arkime-viewer.sh + dest: /services/BambiArkime/arkime-viewer.sh + + - name: Copy init.sh + copy: + src: init.sh + dest: /services/BambiArkime/init.sh diff --git a/ansible/roles/wireguard_arkime/tasks/main.yml b/ansible/roles/wireguard_arkime/tasks/main.yml new file mode 100644 index 0000000..87246fb --- /dev/null +++ b/ansible/roles/wireguard_arkime/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/arkimes/" + dest: /etc/wireguard/ diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index bc84492..7add56b 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -19,15 +19,17 @@ def main() -> None: parser.add_argument("--dns", type=str, default=None) parser.add_argument("--routers", type=int, default=2) parser.add_argument("--checkers", type=int, default=10) + parser.add_argument("--arkimes", type=int, default=2) args = parser.parse_args() teams: int = args.teams dns: Optional[str] = args.dns routers: int = args.routers checkers: int = args.checkers + arkimes: int = args.arkimes logger.info(f"Generating for {teams} teams, {routers} routers and {checkers} checkers") prepare_directories(teams) gen_wireguard_game(teams, dns, routers) - gen_wireguard_internal(teams, checkers, routers) + gen_wireguard_internal(teams, checkers, routers, arkimes) gen_passwords(teams) if dns: gen_userdata_portal(teams) @@ -38,6 +40,7 @@ def prepare_directories(teams: int) -> None: Path(f"{DATA_DIR}/wg_game/").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/wg_internal/").mkdir(parents=True, exist_ok=True) shutil.rmtree(DATA_DIR / "export", ignore_errors=True) + Path(f"{DATA_DIR}/export/ansible/arkimes").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/routers").mkdir(parents=True, exist_ok=True) for team in range(1, teams + 1): diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index aca0c50..c6c774f 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -10,7 +10,7 @@ TEAM_IP_WG_SUBNET, WG_LISTEN_PORT_GAME, Peer, - TeamConfig, + WireguardTeamConfig, WireguardRouterConfig, create_config_file, get_router_cidr_game, @@ -26,8 +26,8 @@ def gen_wireguard_game(teams: int, dns: Optional[str], routers: int) -> None: Generates the game wireguard config files as needed, reusing keys (if present) """ router_configs: list[WireguardRouterConfig] = [] - team_portal_configs: list[TeamConfig] = [] - team_terraform_configs: list[TeamConfig] = [] + team_portal_configs: list[WireguardTeamConfig] = [] + team_terraform_configs: list[WireguardTeamConfig] = [] # Generate router configs for router_id in range(1, routers + 1): @@ -111,13 +111,13 @@ def gen_team_config( team_id: int, router_config: WireguardRouterConfig, dns_suffix: Optional[str], -) -> TeamConfig: +) -> WireguardTeamConfig: if dns_suffix: endpoint = f"router{router_config.router_id}.{dns_suffix}:{WG_LISTEN_PORT_GAME}" else: endpoint = f"[[ROUTER_ADDRESS_{router_config.router_id}]]:{WG_LISTEN_PORT_GAME}" - return TeamConfig( + return WireguardTeamConfig( private_key=private_key, public_key=public_key, cidr=get_vulnbox_cidr(team_id), diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index f085ca7..fa1fa3f 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -7,10 +7,12 @@ DATA_DIR, WG_LISTEN_PORT_INTERNAL, Peer, + WireguardArkimeConfig, WireguardCheckerConfig, WireguardConfig, WireguardRouterConfig, create_config_file, + get_arkime_cidr, get_checker_cidr, get_router_cidr_internal, get_router_index, @@ -21,7 +23,7 @@ logger = logging.getLogger(__file__) -def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: +def gen_wireguard_internal(teams: int, checkers: int, routers: int, arkimes: int) -> None: """ Generates the internal wireguard config files as needed, reusing keys (if present) """ @@ -57,9 +59,11 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: elk_config.peers.append(engine_peer) engine_config.peers.append(elk_peer) + arkime_configs: list[WireguardArkimeConfig] = [] router_configs: list[WireguardRouterConfig] = [] checker_configs: list[WireguardCheckerConfig] = [] checker_peer_list: list[Peer] = [elk_peer, engine_peer] + arkime_peer_list: list[Peer] = [] for router_id in range(1, routers + 1): local_cidr = get_router_cidr_internal(router_id) private_key, public_key = get_wg(Path(f"wg_internal/router{router_id}.key")) @@ -74,6 +78,7 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: listen_port=WG_LISTEN_PORT_INTERNAL, ) ) + router_configs[0].responsible_ips.append("192.168.2.0/24") # Route traffic to teams through the correct router for team in range(1, teams + 1): @@ -91,6 +96,7 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: elk_config.peers.append(peer) engine_config.peers.append(peer) checker_peer_list.append(peer) + arkime_peer_list.append(peer) for checker in range(1, checkers + 1): private_key, public_key = get_wg(Path(f"wg_internal/checker{checker}.key")) @@ -111,10 +117,31 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: ) elk_config.peers.append(checker_peer) engine_config.peers.append(checker_peer) - for gateway_config in router_configs: - gateway_config.peers.append(checker_peer) + for router_config in router_configs: + router_config.peers.append(checker_peer) checker_configs.append(checker_config) + for arkime in range(1, arkimes + 1): + private_key, public_key = get_wg(Path(f"wg_internal/arkime{arkime}.key")) + arkime_cidr = get_arkime_cidr(arkime) + arkime_config = WireguardArkimeConfig( + arkime_id=arkime, + private_key=private_key, + public_key=public_key, + cidr=get_arkime_cidr(arkime), + peers=arkime_peer_list, + listen_port=None + ) + arkime_peer = Peer( + public_key=public_key, + allowed_ips=[arkime_cidr], + endpoint=None, + comment=f"arkime{arkime}", + ) + for router_config in router_configs: + router_config.peers.append(arkime_peer) + arkime_configs.append(arkime_config) + # Save all to disk for router_config in router_configs: Path( @@ -126,6 +153,11 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: f"{DATA_DIR}/export/ansible/checkers/checker{checker_config.checker_id}.conf" ).write_text(create_config_file(checker_config)) + for arkime_config in arkime_configs: + Path( + f"{DATA_DIR}/export/ansible/arkimes/arkime{arkime_config.arkime_id}.conf" + ).write_text(create_config_file(arkime_config)) + Path(f"{DATA_DIR}/export/ansible/engine.conf").write_text( create_config_file(engine_config) ) diff --git a/configgen/configgen/util.py b/configgen/configgen/util.py index 2a408b0..52be794 100644 --- a/configgen/configgen/util.py +++ b/configgen/configgen/util.py @@ -33,6 +33,11 @@ class WireguardConfig: listen_port: Optional[int] +@dataclass +class WireguardArkimeConfig(WireguardConfig): + arkime_id: int + + @dataclass class WireguardRouterConfig(WireguardConfig): router_id: int @@ -40,7 +45,7 @@ class WireguardRouterConfig(WireguardConfig): @dataclass -class TeamConfig(WireguardConfig): +class WireguardTeamConfig(WireguardConfig): team_id: int @@ -136,6 +141,10 @@ def get_checker_cidr(checker_id: int) -> str: return f"192.168.1.{checker_id}/32" +def get_arkime_cidr(arkime_id: int) -> str: + return f"192.168.2.{arkime_id}/32" + + def get_router_cidr_game(router_id: int) -> str: return f"10.13.0.{router_id}/32" diff --git a/packer/bambiarkime.json b/packer/bambiarkime.json new file mode 100644 index 0000000..bade5bc --- /dev/null +++ b/packer/bambiarkime.json @@ -0,0 +1,26 @@ +{ + "provisioners": [ + { + "type": "ansible", + "playbook_file": "../ansible/bambiarkime.yml", + "host_alias": "packer-arkime", + "extra_arguments": [ + "--extra-vars", + "ansible_python_interpreter=/usr/bin/python3" + ] + } + ], + "builders": [ + { + "type": "hcloud", + "image": "ubuntu-22.04", + "location": "fsn1", + "server_type": "cx11", + "ssh_username": "root", + "snapshot_name": "bambiarkime-{{timestamp}}", + "snapshot_labels": { + "type": "bambiarkime" + } + } + ] +} diff --git a/terraform/bambiarkime.tf b/terraform/bambiarkime.tf new file mode 100644 index 0000000..07ba8f8 --- /dev/null +++ b/terraform/bambiarkime.tf @@ -0,0 +1,44 @@ +variable "arkime_type" { + type = string + default = "cpx31" + nullable = false +} + +variable "arkime_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambiarkime" { + with_selector = var.arkime_count > 0 ? "type=bambiarkime" : null + name = var.arkime_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hetznerdns_record" "bambarkime_dns" { + count = var.hetznerdns_zone != null ? var.arkime_count : 0 + zone_id = data.hetznerdns_zone.zone[0].id + name = "arkime${count.index + 1}${local.subdomain}" + value = hcloud_server.bambiarkime[count.index].ipv4_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambiarkime" { + count = var.arkime_count + name = "arkime${count.index + 1}" + image = data.hcloud_image.bambiarkime.id + location = var.home_location + server_type = var.arkime_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_arkime.tftpl", { + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", + engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + } + ) +} diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index aba77f7..77f7457 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -1,44 +1,44 @@ -variable "checker_type" { - type = string - default = "cpx31" - nullable = false -} - -variable "checker_count" { - type = number - default = 0 - nullable = false -} - -data "hcloud_image" "bambichecker" { - with_selector = var.checker_count > 0 ? "type=bambichecker" : null - name = var.checker_count > 0 ? null : "debian-10" - most_recent = true -} - -resource "hetznerdns_record" "bambchecker_dns" { - count = var.hetznerdns_zone != null ? var.checker_count : 0 - zone_id = data.hetznerdns_zone.zone[0].id - name = "checker${count.index + 1}${local.subdomain}" - value = hcloud_server.bambichecker[count.index].ipv4_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambichecker" { - count = var.checker_count - name = "checker${count.index + 1}" - image = data.hcloud_image.bambichecker.id - location = var.home_location - server_type = var.checker_type - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_checker.tftpl", { - id = "${count.index + 1}", - router_ips = hcloud_floating_ip.bambirouter_ip, - elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", - engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", - } - ) -} +variable "checker_type" { + type = string + default = "cpx31" + nullable = false +} + +variable "checker_count" { + type = number + default = 0 + nullable = false +} + +data "hcloud_image" "bambichecker" { + with_selector = var.checker_count > 0 ? "type=bambichecker" : null + name = var.checker_count > 0 ? null : "debian-10" + most_recent = true +} + +resource "hetznerdns_record" "bambchecker_dns" { + count = var.hetznerdns_zone != null ? var.checker_count : 0 + zone_id = data.hetznerdns_zone.zone[0].id + name = "checker${count.index + 1}${local.subdomain}" + value = hcloud_server.bambichecker[count.index].ipv4_address + type = "A" + ttl = 60 +} + +resource "hcloud_server" "bambichecker" { + count = var.checker_count + name = "checker${count.index + 1}" + image = data.hcloud_image.bambichecker.id + location = var.home_location + server_type = var.checker_type + ssh_keys = data.hcloud_ssh_keys.all_keys.*.id + + user_data = templatefile( + "user_data_checker.tftpl", { + id = "${count.index + 1}", + router_ips = hcloud_floating_ip.bambirouter_ip, + elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", + engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + } + ) +} diff --git a/terraform/terraform.tfvars.sample b/terraform/terraform.tfvars.sample index 3c83c16..e42a6f7 100644 --- a/terraform/terraform.tfvars.sample +++ b/terraform/terraform.tfvars.sample @@ -1,5 +1,5 @@ hetznerdns_zone = "bambi.ovh" -hetznerdns_suffix = "prod.bambi.ovh" +hetznerdns_suffix = "prod" home_location = "fsn1" router_count = 2 @@ -13,6 +13,3 @@ checker_type = "cpx31" engine_type = "cpx31" elk_type = "cpx31" vulnbox_type = "cpx21" - -vpn_floating_ip_only = true -internal_floating_ip_only = true diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl new file mode 100644 index 0000000..ee867c3 --- /dev/null +++ b/terraform/user_data_arkime.tftpl @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +# Network +mv "/etc/wireguard/arkime${id}.conf" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" +sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" +%{ for router_index, router in router_ips ~} + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" + sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" +%{ endfor ~} +systemctl enable "wg-quick@internal" +systemctl start "wg-quick@internal" + +# Start elasticsearch cluster +sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml + +# Start Arkime cluster +sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/config.ini From aac035852df48cb436fbf80404850c7d71408259 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 19 Nov 2023 19:39:58 +0100 Subject: [PATCH 14/61] lucasreview --- configgen/configgen/__init__.py | 4 +++- configgen/configgen/gen_wireguard_game.py | 2 +- configgen/configgen/gen_wireguard_internal.py | 6 ++++-- terraform/user_data_arkime.tftpl | 3 +-- terraform/user_data_checker.tftpl | 3 +-- terraform/user_data_elk.tftpl | 3 +-- terraform/user_data_engine.tftpl | 3 +-- terraform/user_data_router.tftpl | 6 ++---- terraform/user_data_vulnbox.tftpl | 3 +-- 9 files changed, 15 insertions(+), 18 deletions(-) diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index 7add56b..702ecb8 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -26,7 +26,9 @@ def main() -> None: routers: int = args.routers checkers: int = args.checkers arkimes: int = args.arkimes - logger.info(f"Generating for {teams} teams, {routers} routers and {checkers} checkers") + logger.info( + f"Generating for {teams} teams, {routers} routers and {checkers} checkers" + ) prepare_directories(teams) gen_wireguard_game(teams, dns, routers) gen_wireguard_internal(teams, checkers, routers, arkimes) diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index c6c774f..c9de121 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -10,8 +10,8 @@ TEAM_IP_WG_SUBNET, WG_LISTEN_PORT_GAME, Peer, - WireguardTeamConfig, WireguardRouterConfig, + WireguardTeamConfig, create_config_file, get_router_cidr_game, get_vulnbox_cidr, diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index fa1fa3f..cfe7f80 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -23,7 +23,9 @@ logger = logging.getLogger(__file__) -def gen_wireguard_internal(teams: int, checkers: int, routers: int, arkimes: int) -> None: +def gen_wireguard_internal( + teams: int, checkers: int, routers: int, arkimes: int +) -> None: """ Generates the internal wireguard config files as needed, reusing keys (if present) """ @@ -130,7 +132,7 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int, arkimes: int public_key=public_key, cidr=get_arkime_cidr(arkime), peers=arkime_peer_list, - listen_port=None + listen_port=None, ) arkime_peer = Peer( public_key=public_key, diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl index ee867c3..3c31b2f 100644 --- a/terraform/user_data_arkime.tftpl +++ b/terraform/user_data_arkime.tftpl @@ -9,8 +9,7 @@ sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" %{ endfor ~} -systemctl enable "wg-quick@internal" -systemctl start "wg-quick@internal" +systemctl enable --now "wg-quick@internal" # Start elasticsearch cluster sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml diff --git a/terraform/user_data_checker.tftpl b/terraform/user_data_checker.tftpl index ca8a818..539f733 100644 --- a/terraform/user_data_checker.tftpl +++ b/terraform/user_data_checker.tftpl @@ -9,8 +9,7 @@ sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" %{ endfor ~} -systemctl enable "wg-quick@internal" -systemctl start "wg-quick@internal" +systemctl enable --now "wg-quick@internal" # Checkers for service in $(ls /services/); do diff --git a/terraform/user_data_elk.tftpl b/terraform/user_data_elk.tftpl index 30a4964..03bd860 100644 --- a/terraform/user_data_elk.tftpl +++ b/terraform/user_data_elk.tftpl @@ -17,8 +17,7 @@ sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" %{ endfor ~} -systemctl enable "wg-quick@internal" -systemctl start "wg-quick@internal" +systemctl enable --now "wg-quick@internal" # Checkers for service in $(ls /services/); do diff --git a/terraform/user_data_engine.tftpl b/terraform/user_data_engine.tftpl index 561840d..ea5d0bf 100644 --- a/terraform/user_data_engine.tftpl +++ b/terraform/user_data_engine.tftpl @@ -17,5 +17,4 @@ netplan apply sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" -systemctl enable "wg-quick@internal" -systemctl start "wg-quick@internal" +systemctl enable --now "wg-quick@internal" diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index d5cb937..9d76645 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -26,7 +26,5 @@ mv "/etc/wireguard/router${id}_game.conf" "/etc/wireguard/router.conf" sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" -systemctl enable "wg-quick@internal" -systemctl start "wg-quick@internal" -systemctl enable "wg-quick@router" -systemctl start "wg-quick@router" +systemctl enable --now "wg-quick@internal" +systemctl enable --now "wg-quick@router" diff --git a/terraform/user_data_vulnbox.tftpl b/terraform/user_data_vulnbox.tftpl index 9688996..2fa1047 100644 --- a/terraform/user_data_vulnbox.tftpl +++ b/terraform/user_data_vulnbox.tftpl @@ -8,8 +8,7 @@ EOF %{ for router_index, router in router_ips ~} sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/game.conf" %{ endfor ~} -systemctl enable "wg-quick@game" -systemctl start "wg-quick@game" +systemctl enable --now "wg-quick@game" # Services for service in $(ls /services/); do From 48ecdf1daf63251d7d44b2ff772e32ebe7966402 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 19 Nov 2023 21:02:13 +0100 Subject: [PATCH 15/61] more arkime --- ansible/roles/bambi-arkime/files/Dockerfile | 2 +- .../bambi-arkime/files/docker-entrypoint.sh | 6 ++++++ ansible/roles/bambi-arkime/files/init.sh | 16 ---------------- configgen/configgen/gen_wireguard_internal.py | 4 +++- terraform/bambiarkime.tf | 6 ++++++ terraform/user_data_arkime.tftpl | 13 +++++++++++++ 6 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 ansible/roles/bambi-arkime/files/docker-entrypoint.sh delete mode 100644 ansible/roles/bambi-arkime/files/init.sh diff --git a/ansible/roles/bambi-arkime/files/Dockerfile b/ansible/roles/bambi-arkime/files/Dockerfile index 7ea2a3b..f24b593 100644 --- a/ansible/roles/bambi-arkime/files/Dockerfile +++ b/ansible/roles/bambi-arkime/files/Dockerfile @@ -8,4 +8,4 @@ RUN dpkg -i ./*.deb; apt-get install -fy RUN touch /opt/arkime/etc/oui.txt RUN touch /opt/arkime/etc/ipv4-address-space.csv -ENTRYPOINT ["tail", "-f", "/dev/null"] +ENTRYPOINT /BambiArkime/docker-entrypoint.sh diff --git a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh new file mode 100644 index 0000000..ac4d981 --- /dev/null +++ b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -e + +sh ./arkime-capture.sh bambiarkime1 & +sh ./arkime-viewer.sh bambiarkime1 & +wait diff --git a/ansible/roles/bambi-arkime/files/init.sh b/ansible/roles/bambi-arkime/files/init.sh deleted file mode 100644 index 8eb45d0..0000000 --- a/ansible/roles/bambi-arkime/files/init.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set -e -elasticsearch="192.168.2.1" - -while ! curl -sq http://$elasticsearch:9200; do - echo "Waiting for elasticsearch to start..."; - sleep 3; -done - -echo "Initializing Arkime Elasticsearch" -/opt/arkime/db/db.pl http://$elasticsearch:9200 init -/opt/arkime/bin/arkime_add_user.sh admin "Admin User" admin --admin - -sh ./arkime-capture.sh bambiarkime1 & -sh ./arkime-viewer.sh bambiarkime1 & -wait diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index cfe7f80..a0e7438 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -80,7 +80,9 @@ def gen_wireguard_internal( listen_port=WG_LISTEN_PORT_INTERNAL, ) ) - router_configs[0].responsible_ips.append("192.168.2.0/24") + # TODO replace with peers? + if routers > 0: + router_configs[0].responsible_ips.append("192.168.2.0/24") # Route traffic to teams through the correct router for team in range(1, teams + 1): diff --git a/terraform/bambiarkime.tf b/terraform/bambiarkime.tf index 07ba8f8..0a6244f 100644 --- a/terraform/bambiarkime.tf +++ b/terraform/bambiarkime.tf @@ -10,6 +10,10 @@ variable "arkime_count" { nullable = false } +locals { + subnet = "192.168.2.0/24" +} + data "hcloud_image" "bambiarkime" { with_selector = var.arkime_count > 0 ? "type=bambiarkime" : null name = var.arkime_count > 0 ? null : "debian-10" @@ -36,6 +40,8 @@ resource "hcloud_server" "bambiarkime" { user_data = templatefile( "user_data_arkime.tftpl", { id = "${count.index + 1}", + masters = join(",", [for i in range(var.arkime_count) : cidrhost(local.subnet, i+1)]), + seeds = join(",", setsubtract([for i in range(var.arkime_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) router_ips = hcloud_floating_ip.bambirouter_ip, elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl index 3c31b2f..6594e40 100644 --- a/terraform/user_data_arkime.tftpl +++ b/terraform/user_data_arkime.tftpl @@ -13,6 +13,19 @@ systemctl enable --now "wg-quick@internal" # Start elasticsearch cluster sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +sed -i -e "s#\[\[INITIAL_MASTER_NODES\]\]#${masters}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +sed -i -e "s#\[\[SEED_HOSTS\]\]#${seeds}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +cd /services/BamiArkimeElasticsearch/ +docker compose up -d # Start Arkime cluster sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/config.ini +cd /services/BamiArkime/ +docker compose build +if [ ${id} -eq 1 ] + #/opt/arkime/db/db.pl "http://192.168.2.${id}:9200" init + #/opt/arkime/bin/arkime_add_user.sh admin "Admin User" admin --admin +then +fi + +#docker compose up -d From fffcb9c9d5508a78c2d6332adf1544cce4162c86 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 19 Nov 2023 20:04:01 +0000 Subject: [PATCH 16/61] make sh exec --- ansible/roles/bambi-arkime/files/arkime-capture.sh | 0 ansible/roles/bambi-arkime/files/arkime-viewer.sh | 0 ansible/roles/bambi-arkime/files/docker-entrypoint.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ansible/roles/bambi-arkime/files/arkime-capture.sh mode change 100644 => 100755 ansible/roles/bambi-arkime/files/arkime-viewer.sh mode change 100644 => 100755 ansible/roles/bambi-arkime/files/docker-entrypoint.sh diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/bambi-arkime/files/arkime-capture.sh old mode 100644 new mode 100755 diff --git a/ansible/roles/bambi-arkime/files/arkime-viewer.sh b/ansible/roles/bambi-arkime/files/arkime-viewer.sh old mode 100644 new mode 100755 diff --git a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh old mode 100644 new mode 100755 From f249180679bfa343f8465ec33e4b6d77689d82cb Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 19 Nov 2023 22:02:02 +0100 Subject: [PATCH 17/61] more arki work --- .../bambi-arkime/files/docker-entrypoint.sh | 4 ++-- ansible/roles/bambi-arkime/tasks/main.yml | 6 +++--- terraform/user_data_arkime.tftpl | 16 ++++++++++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh index ac4d981..c6a1e00 100755 --- a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh +++ b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -sh ./arkime-capture.sh bambiarkime1 & -sh ./arkime-viewer.sh bambiarkime1 & +sh ./arkime-capture.sh [[ARKIME]] & +sh ./arkime-viewer.sh [[ARKIME]] & wait diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml index 310a4ce..88d4ba6 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -50,7 +50,7 @@ src: arkime-viewer.sh dest: /services/BambiArkime/arkime-viewer.sh - - name: Copy init.sh + - name: Copy docker-entrypoint.sh copy: - src: init.sh - dest: /services/BambiArkime/init.sh + src: docker-entrypoint.sh + dest: /services/BambiArkime/docker-entrypoint.sh diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl index 6594e40..4de73e3 100644 --- a/terraform/user_data_arkime.tftpl +++ b/terraform/user_data_arkime.tftpl @@ -1,6 +1,9 @@ #!/bin/bash set -e +# TODO make this reboot-safe +iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT + # Network mv "/etc/wireguard/arkime${id}.conf" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" @@ -15,17 +18,22 @@ systemctl enable --now "wg-quick@internal" sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml sed -i -e "s#\[\[INITIAL_MASTER_NODES\]\]#${masters}#g" /services/BambiArkimeElasticsearch/docker-compose.yml sed -i -e "s#\[\[SEED_HOSTS\]\]#${seeds}#g" /services/BambiArkimeElasticsearch/docker-compose.yml -cd /services/BamiArkimeElasticsearch/ +cd /services/BambiArkimeElasticsearch/ docker compose up -d # Start Arkime cluster sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/config.ini -cd /services/BamiArkime/ +sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/docker-entrypoint.sh +cd /services/BambiArkime/ docker compose build + + if [ ${id} -eq 1 ] - #/opt/arkime/db/db.pl "http://192.168.2.${id}:9200" init - #/opt/arkime/bin/arkime_add_user.sh admin "Admin User" admin --admin then + docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://192.168.2.${id}:9200" init + docker compose run --entrypoint="/opt/arkime/bin/arkime_add_user.sh" arkime admin "Admin" admin --admin + # TODO create marker fi +# TODO wait for marker #docker compose up -d From 8cecea5c04d12576d9c8ddc985c3e3d94ef4fc9c Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 19 Nov 2023 23:12:53 +0100 Subject: [PATCH 18/61] fix arkime viewer connectivity (but not funktionality -.-) --- ansible/roles/bambi-arkime/files/arkime-capture.sh | 2 +- ansible/roles/bambi-arkime/files/arkime-viewer.sh | 2 +- ansible/roles/bambi-arkime/files/docker-compose.arkime.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/bambi-arkime/files/arkime-capture.sh index e8a8de3..9ee5f07 100755 --- a/ansible/roles/bambi-arkime/files/arkime-capture.sh +++ b/ansible/roles/bambi-arkime/files/arkime-capture.sh @@ -2,6 +2,6 @@ cd /opt/arkime while : do echo "Starting Arkime capture" - /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /opt/arkime/raw -m --host $1 + /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /opt/arkime/raw -m --host $1 -n $1 sleep 5 done diff --git a/ansible/roles/bambi-arkime/files/arkime-viewer.sh b/ansible/roles/bambi-arkime/files/arkime-viewer.sh index 3c14f4e..b8e06a4 100755 --- a/ansible/roles/bambi-arkime/files/arkime-viewer.sh +++ b/ansible/roles/bambi-arkime/files/arkime-viewer.sh @@ -2,6 +2,6 @@ cd /opt/arkime/viewer while : do echo "Starting Arkime viewer" - /opt/arkime/bin/node viewer.js -c /opt/arkime/etc/config.ini --host $1 + /opt/arkime/bin/node viewer.js -c /opt/arkime/etc/config.ini --host $1 -n $1 sleep 5 done diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml index 46bc932..a9e93fd 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml @@ -4,7 +4,7 @@ services: build: . restart: unless-stopped ports: - - 80:8005 + - 8005:8005 volumes: - "/pcaps:/opt/arkime/raw" - ".:/BambiArkime:ro" From 6f8c71ebd93a65cc1e7274be4d6288494ef0f7b9 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Mon, 20 Nov 2023 12:21:48 +0100 Subject: [PATCH 19/61] start arkime automagically, make arkime truly optional --- ansible/roles/bambi-arkime/tasks/main.yml | 3 +++ terraform/bambiarkime.tf | 18 +++++++++--------- terraform/user_data_arkime.tftpl | 14 +++++++++++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml index 88d4ba6..ec00828 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -44,13 +44,16 @@ copy: src: arkime-capture.sh dest: /services/BambiArkime/arkime-capture.sh + mode: '755' - name: Copy arkime-viewer.sh copy: src: arkime-viewer.sh dest: /services/BambiArkime/arkime-viewer.sh + mode: '755' - name: Copy docker-entrypoint.sh copy: src: docker-entrypoint.sh dest: /services/BambiArkime/docker-entrypoint.sh + mode: '755' diff --git a/terraform/bambiarkime.tf b/terraform/bambiarkime.tf index 0a6244f..dab3125 100644 --- a/terraform/bambiarkime.tf +++ b/terraform/bambiarkime.tf @@ -4,9 +4,9 @@ variable "arkime_type" { nullable = false } -variable "arkime_count" { - type = number - default = 0 +variable "use_arkime" { + type = bool + default = false nullable = false } @@ -15,13 +15,13 @@ locals { } data "hcloud_image" "bambiarkime" { - with_selector = var.arkime_count > 0 ? "type=bambiarkime" : null - name = var.arkime_count > 0 ? null : "debian-10" + with_selector = var.use_arkime ? "type=bambiarkime" : null + name = var.use_arkime ? null : "debian-10" most_recent = true } resource "hetznerdns_record" "bambarkime_dns" { - count = var.hetznerdns_zone != null ? var.arkime_count : 0 + count = var.hetznerdns_zone != null && var.use_arkime ? var.router_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "arkime${count.index + 1}${local.subdomain}" value = hcloud_server.bambiarkime[count.index].ipv4_address @@ -30,7 +30,7 @@ resource "hetznerdns_record" "bambarkime_dns" { } resource "hcloud_server" "bambiarkime" { - count = var.arkime_count + count = var.use_arkime ? var.router_count : 0 name = "arkime${count.index + 1}" image = data.hcloud_image.bambiarkime.id location = var.home_location @@ -40,8 +40,8 @@ resource "hcloud_server" "bambiarkime" { user_data = templatefile( "user_data_arkime.tftpl", { id = "${count.index + 1}", - masters = join(",", [for i in range(var.arkime_count) : cidrhost(local.subnet, i+1)]), - seeds = join(",", setsubtract([for i in range(var.arkime_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) + masters = join(",", [for i in range(var.router_count) : cidrhost(local.subnet, i+1)]), + seeds = join(",", setsubtract([for i in range(var.router_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) router_ips = hcloud_floating_ip.bambirouter_ip, elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl index 4de73e3..ed4ffcb 100644 --- a/terraform/user_data_arkime.tftpl +++ b/terraform/user_data_arkime.tftpl @@ -27,13 +27,21 @@ sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/docker-entr cd /services/BambiArkime/ docker compose build +while ! curl -sq "http://192.168.2.${id}:9200"; do + echo "Waiting for elastic search to start..."; + sleep 3; +done if [ ${id} -eq 1 ] then docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://192.168.2.${id}:9200" init docker compose run --entrypoint="/opt/arkime/bin/arkime_add_user.sh" arkime admin "Admin" admin --admin - # TODO create marker + curl -X PUT "http://192.168.2.${id}:9200/bambi-arkime-init?pretty" fi -# TODO wait for marker -#docker compose up -d +while ! curl -s --head --show-error --fail "http://192.168.2.${id}:9200/bambi-arkime-init/"; do + echo "Waiting for marker index to exist..."; + sleep 3; +done + +docker compose up -d From 448d757c2192159deb841fcd9df53ce0b3a0a8fe Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 24 Nov 2023 22:44:39 +0100 Subject: [PATCH 20/61] towards openvpn --- Dockerfile | 2 +- ansible/bambirouter.yml | 2 +- ansible/bambivulnbox.yml | 1 - .../bambi-openvpn-team-server/tasks/main.yml | 42 ----- ansible/roles/teamvpn-configs/tasks/main.yml | 7 +- configgen/configgen/__init__.py | 12 +- configgen/configgen/gen_openvpn.py | 158 ++++++++++++++++++ configgen/configgen/gen_wireguard_game.py | 4 +- configgen/configgen/gen_wireguard_internal.py | 21 ++- configgen/configgen/util.py | 28 +++- terraform/bambi.tf | 2 +- terraform/bambiarkime.tf | 15 ++ terraform/user_data_arkime.tftpl | 14 ++ 13 files changed, 239 insertions(+), 69 deletions(-) delete mode 100644 ansible/roles/bambi-openvpn-team-server/tasks/main.yml create mode 100644 configgen/configgen/gen_openvpn.py diff --git a/Dockerfile b/Dockerfile index d4f255d..2670f91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get install -y --no-install-recommends rsync git less tmux python3 curl software-properties-common gpg-agent # for ansible and packer install # Poetry -RUN pip install poetry && poetry config virtualenvs.in-project true +RUN pip install poetry && poetry config virtualenvs.in-project false # Ansible RUN add-apt-repository --yes --update ppa:ansible/ansible && apt-get install -y ansible diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index aa20523..e460f96 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -23,4 +23,4 @@ program_list: - "tmux" - "git" - #- teamvpn-configs + - teamvpn-configs diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index fed8949..9678fd2 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -25,4 +25,3 @@ vars: pip_list: - pwntools - #- bambi-openvpn-team-server diff --git a/ansible/roles/bambi-openvpn-team-server/tasks/main.yml b/ansible/roles/bambi-openvpn-team-server/tasks/main.yml deleted file mode 100644 index 623fa3a..0000000 --- a/ansible/roles/bambi-openvpn-team-server/tasks/main.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- - - name: "Install required apt packages" - apt: - package: - - openvpn - - - name: "Enable ipv4 forward" - sysctl: - name: net.ipv4.ip_forward - value: '1' - sysctl_set: yes - - - name: allow related/established forward traffic - iptables: - chain: FORWARD - ctstate: [RELATED, ESTABLISHED] - jump: ACCEPT - - - name: "Allow traffic from team to game" - iptables: - chain: FORWARD - in_interface: team - out_interface: game - jump: ACCEPT - - - name: masquerade outgoing traffic on the game interface - iptables: - table: nat - chain: POSTROUTING - out_interface: game - jump: MASQUERADE - - - name: allow OpenVPN input traffic - iptables: - chain: INPUT - in_interface: eth0 - destination_port: "1194" - protocol: udp - jump: ACCEPT - - - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" diff --git a/ansible/roles/teamvpn-configs/tasks/main.yml b/ansible/roles/teamvpn-configs/tasks/main.yml index 6060930..57e2395 100644 --- a/ansible/roles/teamvpn-configs/tasks/main.yml +++ b/ansible/roles/teamvpn-configs/tasks/main.yml @@ -2,13 +2,12 @@ - name: "Install openvpn" apt: package: - - openvpn - - unzip + - openvpn - name: copy zipped team VPN configs copy: - src: "{{ playbook_dir }}/../config/openvpn_configs/zips" - dest: /root + src: "{{ playbook_dir }}/../config/export/ansible/routers/openvpn/" + dest: /etc/openvpn/server - name: allow openvpn input traffic iptables: diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index 702ecb8..b23b1ac 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Optional +from configgen.gen_openvpn import gen_openvpn from configgen.gen_wireguard_game import gen_wireguard_game from configgen.gen_wireguard_internal import gen_wireguard_internal from configgen.util import DATA_DIR @@ -35,16 +36,21 @@ def main() -> None: gen_passwords(teams) if dns: gen_userdata_portal(teams) + gen_openvpn(teams, routers, dns) def prepare_directories(teams: int) -> None: Path(f"{DATA_DIR}/passwords/").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/wg_game/").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/wg_internal/").mkdir(parents=True, exist_ok=True) + for team in range(1, teams + 1): + Path(f"{DATA_DIR}/openvpn/team{team}/").mkdir(parents=True, exist_ok=True) shutil.rmtree(DATA_DIR / "export", ignore_errors=True) Path(f"{DATA_DIR}/export/ansible/arkimes").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) - Path(f"{DATA_DIR}/export/ansible/routers").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/export/ansible/routers/openvpn").mkdir( + parents=True, exist_ok=True + ) for team in range(1, teams + 1): Path(f"{DATA_DIR}/export/portal/team{team}").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/terraform/team{team}").mkdir( @@ -95,7 +101,3 @@ def gen_passwords(teams: int) -> None: pw_file.write_text(secrets.token_hex(32)) export_file = Path(f"{DATA_DIR}/export/portal/team{team}/password.txt") export_file.write_text(pw_file.read_text()) - - -def gen_openvpn() -> None: - pass diff --git a/configgen/configgen/gen_openvpn.py b/configgen/configgen/gen_openvpn.py new file mode 100644 index 0000000..12cdccd --- /dev/null +++ b/configgen/configgen/gen_openvpn.py @@ -0,0 +1,158 @@ +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +from configgen.util import DATA_DIR, _get_team_octets, run_subprocess + +logger = logging.getLogger(__file__) + + +@dataclass +class OpenVPNTeamConfig: + client_key: str + server_key: str + client_crt: str + server_crt: str + ca_crt: str + ta_key: str + + +def gen_openvpn(teams: int, routers: int, dns: str) -> None: + dh = get_dh() + for team in range(1, teams + 1): + config = get_team_config(team) + port = 30000 + team + x, y = _get_team_octets(team) + TEAM_SUBNET_PREFIX = f"10.{x}.{y}" + server_config = f"""port {port} +local 0.0.0.0 +proto udp +dev team{team} +dev-type tun + +mode server +tls-server +key-direction 0 +cipher AES-256-CBC +auth SHA256 + +topology subnet +ifconfig {TEAM_SUBNET_PREFIX}.129 255.255.255.128 +push "topology subnet" +ifconfig-pool {TEAM_SUBNET_PREFIX}.130 {TEAM_SUBNET_PREFIX}.254 +route-gateway {TEAM_SUBNET_PREFIX}.129 +push "route-gateway {TEAM_SUBNET_PREFIX}.129" +push "route 10.0.0.0 255.0.0.0" + +keepalive 10 120 +persist-key +persist-tun +status /etc/openvpn/server/team{team}.log +verb 3 + +duplicate-cn + +<ca> +{config.ca_crt}</ca> + +<cert> +{config.server_crt}</cert> + +<key> +{config.server_key}</key> + +<tls-auth> +{config.ta_key}</tls-auth> + +<dh> +{dh}</dh> +""" + client_config = f"""proto udp +dev tun +remote router{(team - 1) % routers + 1}.{dns} {port} +resolv-retry infinite +nobind + +tls-client +remote-cert-tls server +key-direction 1 +cipher AES-256-CBC +auth SHA256 + +keepalive 10 120 +verb 3 +pull + +<ca> +{config.ca_crt}</ca> + +<tls-auth> +{config.ta_key}</tls-auth> + +<cert> +{config.client_crt}</cert> + +<key> +{config.client_key}</key> +""" + # Save to disk + Path(f"{DATA_DIR}/export/ansible/routers/openvpn/team{team}.conf").write_text( + server_config + ) + Path(f"{DATA_DIR}/export/portal/team{team}/client.ovpn").write_text( + client_config + ) + + +def get_dh() -> str: + dh_file = Path(f"{DATA_DIR}/openvpn/dh.pem") + try: + return dh_file.read_text() + except FileNotFoundError: + pass + + logger.debug(f"Generating {dh_file}") + run_subprocess(["openssl", "dhparam", "-out", "dh.pem", "2048"]) + return dh_file.read_text() + + +def get_team_config(team: int) -> OpenVPNTeamConfig: + files = [ + Path(f"{DATA_DIR}/openvpn/team{team}/pki/private/client.key"), + Path(f"{DATA_DIR}/openvpn/team{team}/pki/private/server.key"), + Path(f"{DATA_DIR}/openvpn/team{team}/pki/issued/client.crt"), + Path(f"{DATA_DIR}/openvpn/team{team}/pki/issued/server.crt"), + Path(f"{DATA_DIR}/openvpn/team{team}/pki/ca.crt"), + Path(f"{DATA_DIR}/openvpn/team{team}/ta.key"), + ] + if all(list(map(os.path.isfile, files))): + return OpenVPNTeamConfig( + files[0].read_text(), + files[1].read_text(), + files[2].read_text(), + files[3].read_text(), + files[4].read_text(), + files[5].read_text(), + ) + + # TODO clean directory (might have half-done state which makes easyrsa ask things interactively) + logger.debug(f"Generating fresh openvpn for team{team}") + old_cwd = os.getcwd() + os.chdir(f"{DATA_DIR}/openvpn/team{team}") + run_subprocess(["easyrsa", "init-pki"], "yes\n") + run_subprocess(["easyrsa", "build-ca", "nopass"], "CA\n") + run_subprocess(["easyrsa", "gen-req", "server", "nopass"], "server\n") + run_subprocess(["easyrsa", "sign-req", "server", "server"], "yes\n") + run_subprocess(["easyrsa", "gen-req", "client", "nopass"], "client\n") + run_subprocess(["easyrsa", "sign-req", "client", "client"], "yes\n") + run_subprocess(["openvpn", "--genkey", "secret", "ta.key"]) + os.chdir(old_cwd) + return OpenVPNTeamConfig( + files[0].read_text(), + files[1].read_text(), + files[2].read_text(), + files[3].read_text(), + files[4].read_text(), + files[5].read_text(), + ) diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index c9de121..49be354 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -51,7 +51,7 @@ def gen_wireguard_game(teams: int, dns: Optional[str], routers: int) -> None: x, y = (team // 250), (team % 250) router_index = (team - 1) % routers router_config = router_configs[router_index] - if dns != None: + if dns is not None: team_portal_configs.append( gen_team_config(private_key, public_key, team, router_config, dns) ) @@ -88,7 +88,7 @@ def gen_wireguard_game(teams: int, dns: Optional[str], routers: int) -> None: ) ) - if dns != None: + if dns is not None: for team_portal_config in team_portal_configs: Path( f"{DATA_DIR}/export/portal/team{team_portal_config.team_id}/game.conf" diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index a0e7438..bc5ce0e 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -80,9 +80,6 @@ def gen_wireguard_internal( listen_port=WG_LISTEN_PORT_INTERNAL, ) ) - # TODO replace with peers? - if routers > 0: - router_configs[0].responsible_ips.append("192.168.2.0/24") # Route traffic to teams through the correct router for team in range(1, teams + 1): @@ -133,7 +130,7 @@ def gen_wireguard_internal( private_key=private_key, public_key=public_key, cidr=get_arkime_cidr(arkime), - peers=arkime_peer_list, + peers=arkime_peer_list.copy(), listen_port=None, ) arkime_peer = Peer( @@ -146,6 +143,22 @@ def gen_wireguard_internal( router_config.peers.append(arkime_peer) arkime_configs.append(arkime_config) + # Add arkime peering (for elasticseach traffic) + for i in range(arkimes): + for j in range(arkimes): + if i == j: + continue + arkime_i = arkime_configs[i] + arkime_j = arkime_configs[j] + arkime_i.peers.append( + Peer( + public_key=arkime_j.public_key, + allowed_ips=[arkime_j.cidr], + endpoint=f"[[ARKIME_ADDRESS_{arkime_j.arkime_id}]]:{WG_LISTEN_PORT_INTERNAL}", + comment=f"arkime{arkime_j.arkime_id}", + ) + ) + # Save all to disk for router_config in router_configs: Path( diff --git a/configgen/configgen/util.py b/configgen/configgen/util.py index 52be794..e9812cb 100644 --- a/configgen/configgen/util.py +++ b/configgen/configgen/util.py @@ -55,16 +55,28 @@ class WireguardCheckerConfig(WireguardConfig): def run_subprocess(args: list[str], input: Optional[str] = None) -> bytes: - return subprocess.run(args, stdout=subprocess.PIPE, input=input).stdout + if input: + bytes_input = input.encode() + else: + bytes_input = None + result = subprocess.run( + args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, input=bytes_input + ) + if result.returncode != 0: + raise Exception(f"{args} returned {result.returncode}") + return result.stdout -def gen_wg_priv() -> str: +def exec_openvpn_genkey_secret() -> str: + return run_subprocess(["openvpn", "--genkey", "secret"]).strip().decode() + + +def exec_wg_priv() -> str: return run_subprocess(["wg", "genkey"]).strip().decode() -def gen_wg_pub(priv: str) -> str: - return run_subprocess(["wg", "pubkey"], input=priv.encode()).strip().decode() # type: ignore - # TODO fix typeshed? +def exec_wg_pub(priv: str) -> str: + return run_subprocess(["wg", "pubkey"], input=priv).strip().decode() def create_config_file(config: WireguardConfig) -> str: @@ -104,14 +116,14 @@ def get_wg(path: Path) -> Tuple[str, str]: relative_path = DATA_DIR / path try: priv = relative_path.read_text() - return priv, gen_wg_pub(priv) + return priv, exec_wg_pub(priv) except FileNotFoundError: pass logger.debug(f"Generating fresh privkey for {path}") - priv = gen_wg_priv() + priv = exec_wg_priv() relative_path.write_text(priv) - return priv, gen_wg_pub(priv) + return priv, exec_wg_pub(priv) def _get_team_octets(team_id: int) -> Tuple[int, int]: diff --git a/terraform/bambi.tf b/terraform/bambi.tf index 4dc10e4..2d23c9b 100644 --- a/terraform/bambi.tf +++ b/terraform/bambi.tf @@ -36,7 +36,7 @@ variable "hetznerdns_zone" { variable "subdomain" { type = string - default = "" + default = null } locals { diff --git a/terraform/bambiarkime.tf b/terraform/bambiarkime.tf index dab3125..f8380f5 100644 --- a/terraform/bambiarkime.tf +++ b/terraform/bambiarkime.tf @@ -20,6 +20,19 @@ data "hcloud_image" "bambiarkime" { most_recent = true } +resource "hcloud_floating_ip" "bambiarkime_ip" { + count = var.use_arkime ? var.router_count : 0 + type = "ipv4" + name = "arkime${count.index + 1}" + home_location = var.home_location +} + +resource "hcloud_floating_ip_assignment" "bambiarkime_ipa" { + count = var.use_arkime ? var.router_count : 0 + floating_ip_id = hcloud_floating_ip.bambiarkime_ip[count.index].id + server_id = hcloud_server.bambiarkime[count.index].id +} + resource "hetznerdns_record" "bambarkime_dns" { count = var.hetznerdns_zone != null && var.use_arkime ? var.router_count : 0 zone_id = data.hetznerdns_zone.zone[0].id @@ -39,12 +52,14 @@ resource "hcloud_server" "bambiarkime" { user_data = templatefile( "user_data_arkime.tftpl", { + index = count.index, id = "${count.index + 1}", masters = join(",", [for i in range(var.router_count) : cidrhost(local.subnet, i+1)]), seeds = join(",", setsubtract([for i in range(var.router_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) router_ips = hcloud_floating_ip.bambirouter_ip, elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + arkime_ips = hcloud_floating_ip.bambiarkime_ip, } ) } diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl index ed4ffcb..f968bb8 100644 --- a/terraform/user_data_arkime.tftpl +++ b/terraform/user_data_arkime.tftpl @@ -1,6 +1,16 @@ #!/bin/bash set -e +cat > /etc/netplan/60-floating-ip.yaml <<EOF +network: + version: 2 + ethernets: + eth0: + addresses: + - ${arkime_ips[index].ip_address}/32 +EOF +netplan apply + # TODO make this reboot-safe iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT @@ -12,6 +22,10 @@ sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" %{ endfor ~} +%{ for arkime_index, arkime in arkime_ips ~} + sed -i -e "s#\[\[ARKIME_ADDRESS_${arkime_index+1}\]\]#${arkime.ip_address}#g" "/etc/wireguard/internal.conf" + sed -i -e "s#\[\[ARKIME_ADDRESS_${arkime_index+1}\]\]#${arkime.ip_address}#g" "/etc/wireguard/internal.conf" +%{ endfor ~} systemctl enable --now "wg-quick@internal" # Start elasticsearch cluster From 37c80cef04513c5e9ae959197295c9058c471dba Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 24 Nov 2023 22:52:16 +0100 Subject: [PATCH 21/61] generate dh at proper location --- configgen/configgen/gen_openvpn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configgen/configgen/gen_openvpn.py b/configgen/configgen/gen_openvpn.py index 12cdccd..90e5e63 100644 --- a/configgen/configgen/gen_openvpn.py +++ b/configgen/configgen/gen_openvpn.py @@ -113,7 +113,7 @@ def get_dh() -> str: pass logger.debug(f"Generating {dh_file}") - run_subprocess(["openssl", "dhparam", "-out", "dh.pem", "2048"]) + run_subprocess(["openssl", "dhparam", "-out", f"{DATA_DIR}/openvpn/dh.pem", "2048"]) return dh_file.read_text() From 752e0491933d89b716ae963de4c7d4d5f7929420 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 15 Dec 2023 17:27:02 +0100 Subject: [PATCH 22/61] old arkime setup --- ansible/bambirouter.yml | 53 +++--- .../bambi-arkime/files/arkime-capture.sh | 2 +- ansible/roles/bambi-arkime/files/config.ini | 2 +- .../files/docker-compose.arkime.yml | 2 +- ansible/roles/bambi-arkime/tasks/main.yml | 154 +++++++++++------- .../files/enorouterdump.service | 14 ++ .../router_trafficcapture/tasks/main.yml | 124 ++++++++++++++ configgen/configgen/gen_wireguard_internal.py | 2 +- terraform/bambiarkime.tf | 2 +- terraform/user_data_arkime.tftpl | 13 +- 10 files changed, 273 insertions(+), 95 deletions(-) create mode 100644 ansible/roles/router_trafficcapture/files/enorouterdump.service create mode 100644 ansible/roles/router_trafficcapture/tasks/main.yml diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index e460f96..867842b 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -1,26 +1,27 @@ ---- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - roles: - - role: filebeat - vars: - elk: 192.168.3.0 - - role: journalbeat - vars: - elk: 192.168.3.0 - - role: metricbeat - vars: - elk: 192.168.3.0 - - bambi-ssh-keys - - firewall - - bambi-wireguard-router - - role: programs - vars: - program_list: - - "tmux" - - "git" - - teamvpn-configs +--- + - hosts: all + become: yes + become_method: sudo + vars_files: + - config_bambi.yml + + roles: + - role: filebeat + vars: + elk: 192.168.3.0 + - role: journalbeat + vars: + elk: 192.168.3.0 + - role: metricbeat + vars: + elk: 192.168.3.0 + - bambi-ssh-keys + - firewall + - bambi-wireguard-router + - router_trafficcapture + - role: programs + vars: + program_list: + - "tmux" + - "git" + - teamvpn-configs diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/bambi-arkime/files/arkime-capture.sh index 9ee5f07..e32c912 100755 --- a/ansible/roles/bambi-arkime/files/arkime-capture.sh +++ b/ansible/roles/bambi-arkime/files/arkime-capture.sh @@ -2,6 +2,6 @@ cd /opt/arkime while : do echo "Starting Arkime capture" - /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /opt/arkime/raw -m --host $1 -n $1 + /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /pcaps/pcaps -m --host $1 -n $1 sleep 5 done diff --git a/ansible/roles/bambi-arkime/files/config.ini b/ansible/roles/bambi-arkime/files/config.ini index 0bd56a3..2649431 100644 --- a/ansible/roles/bambi-arkime/files/config.ini +++ b/ansible/roles/bambi-arkime/files/config.ini @@ -11,7 +11,7 @@ httpRealm=Moloch # Semicolon ';' seperated list of interfaces to listen on for traffic interface=eth1 # The directory to save raw pcap files to -pcapDir=/opt/arkime/raw +pcapDir=/pcaps/pcaps # The max raw pcap file size in gigabytes, with a max value of 36G. # The disk should have room for at least 10*maxFileSizeG diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml index a9e93fd..c9ad6a2 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml @@ -6,6 +6,6 @@ services: ports: - 8005:8005 volumes: - - "/pcaps:/opt/arkime/raw" + - "/pcaps:/pcaps" - ".:/BambiArkime:ro" - "./config.ini:/opt/arkime/etc/config.ini:ro" diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml index ec00828..8bf3d10 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -1,59 +1,97 @@ --- - - name: Increase max_map_count - sysctl: - name: vm.max_map_count - value: "262144" - sysctl_set: yes - - - name: Create /services/BambiArkimeElasticsearch - file: - path: /services/BambiArkimeElasticsearch - state: directory - - - name: docker-compose.elasticsearch.yml - copy: - src: docker-compose.elasticsearch.yml - dest: /services/BambiArkimeElasticsearch/docker-compose.yml - - - name: Pull ES - shell: docker compose pull - args: - chdir: /services/BambiArkimeElasticsearch - - - name: Create /services/BambiArkime - file: - path: /services/BambiArkime - state: directory - - - name: Copy docker-compose.arkime.yml - copy: - src: docker-compose.arkime.yml - dest: /services/BambiArkime/docker-compose.yml - - - name: Copy config.ini - copy: - src: config.ini - dest: /services/BambiArkime/config.ini - - - name: Copy Dockerfile - copy: - src: Dockerfile - dest: /services/BambiArkime/Dockerfile - - - name: Copy arkime-capture.sh - copy: - src: arkime-capture.sh - dest: /services/BambiArkime/arkime-capture.sh - mode: '755' - - - name: Copy arkime-viewer.sh - copy: - src: arkime-viewer.sh - dest: /services/BambiArkime/arkime-viewer.sh - mode: '755' - - - name: Copy docker-entrypoint.sh - copy: - src: docker-entrypoint.sh - dest: /services/BambiArkime/docker-entrypoint.sh - mode: '755' +- name: allow wireguard input traffic (internal) + iptables: + chain: INPUT + in_interface: eth0 + destination_port: "51821" + protocol: udp + jump: ACCEPT + +# TODO fixme @ldruschk +- shell: iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT +- shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 9300 -j ACCEPT + +- name: persist iptables config + shell: "iptables-save > /etc/iptables/rules.v4" + +- name: Increase max_map_count + sysctl: + name: vm.max_map_count + value: "262144" + sysctl_set: yes + +- name: Create /services/BambiArkimeElasticsearch + file: + path: /services/BambiArkimeElasticsearch + state: directory + +- name: docker-compose.elasticsearch.yml + copy: + src: docker-compose.elasticsearch.yml + dest: /services/BambiArkimeElasticsearch/docker-compose.yml + +- name: Pull ES + shell: docker compose pull + args: + chdir: /services/BambiArkimeElasticsearch + +- name: Create /services/BambiArkime + file: + path: /services/BambiArkime + state: directory + +- name: Copy docker-compose.arkime.yml + copy: + src: docker-compose.arkime.yml + dest: /services/BambiArkime/docker-compose.yml + +- name: Copy config.ini + copy: + src: config.ini + dest: /services/BambiArkime/config.ini + +- name: Copy Dockerfile + copy: + src: Dockerfile + dest: /services/BambiArkime/Dockerfile + +- name: Copy arkime-capture.sh + copy: + src: arkime-capture.sh + dest: /services/BambiArkime/arkime-capture.sh + mode: '755' + +- name: Copy arkime-viewer.sh + copy: + src: arkime-viewer.sh + dest: /services/BambiArkime/arkime-viewer.sh + mode: '755' + +- name: Copy docker-entrypoint.sh + copy: + src: docker-entrypoint.sh + dest: /services/BambiArkime/docker-entrypoint.sh + mode: '755' + +# - name: build +# shell: "docker compose build" +# args: +# chdir: /services/BambiArkime + +- name: "install autofs and nfs-client" + apt: + name: + - autofs + - nfs-client + +- name: "create /etc/auto.nfs" + shell: echo "pcaps 192.168.0.[[ARKIME_ID]]:/pcaps" >> /etc/auto.nfs + +- lineinfile: + path: /etc/auto.master + line: "/pcaps /etc/auto.nfs" + +- name: "Create /pcaps directory" + file: + path: /pcaps + state: directory diff --git a/ansible/roles/router_trafficcapture/files/enorouterdump.service b/ansible/roles/router_trafficcapture/files/enorouterdump.service new file mode 100644 index 0000000..5f3f5aa --- /dev/null +++ b/ansible/roles/router_trafficcapture/files/enorouterdump.service @@ -0,0 +1,14 @@ +[Unit] +Description=enorouterdump packet capture +After=docker.service containerd.service wg-quick@router.service + +[Service] +Restart=always +RestartSec=1 +TimeoutStartSec=300 +WorkingDirectory=/pcaps + +ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/router_trafficcapture/tasks/main.yml b/ansible/roles/router_trafficcapture/tasks/main.yml new file mode 100644 index 0000000..17bd4b4 --- /dev/null +++ b/ansible/roles/router_trafficcapture/tasks/main.yml @@ -0,0 +1,124 @@ +--- +- name: install nfs-server + apt: + name: + - nfs-server + +- name: Enable enorouterdump + service: + name: nfs-server + enabled: yes + +- name: "Create /pcaps directory" + file: + path: /pcaps + state: directory + +- name: Install tcpdump + apt: + name: tcpdump + state: present + update_cache: yes + +- name: Copy over the enorouterdump unit + copy: + src: enorouterdump.service + dest: /etc/systemd/system/enorouterdump.service + +- name: Disable tcpdump apparmor profile + file: + path: /etc/apparmor.d/disable/usr.sbin.tcpdump + src: /etc/apparmor.d/usr.sbin.tcpdump + state: link + force: yes + follow: no + +- name: Reload system daemon + systemd: + name: "enorouterdump" # ansible <2.4 always requires 'name' + daemon_reload: yes + +- name: Enable enorouterdump + service: + name: enorouterdump + enabled: yes + +- name: "add pcaps to /etc/exports" + lineinfile: + path: /etc/exports + line: "/pcaps 192.168.2.0/255.255.255.0(ro,subtree_check)" + +- name: "run exportfs -a" + shell: exportfs -a + +- name: "specify mountd port to always be 33599" + lineinfile: + path: /etc/default/nfs-kernel-server + regexp: "^RPCMOUNTDOPTS" + line: 'RPCMOUNTDOPTS="--manage-gids --port 33599"' + +- name: allow NFS traffic (tcp port 111) + iptables: + chain: INPUT + in_interface: internal + protocol: tcp + source: "192.168.2.0/24" + destination_port: "111" + ctstate: NEW + syn: match + jump: ACCEPT + +- name: allow NFS traffic (tcp port 2049) + iptables: + chain: INPUT + in_interface: internal + protocol: tcp + source: "192.168.2.0/24" + destination_port: "2049" + ctstate: NEW + syn: match + jump: ACCEPT + +- name: allow NFS traffic (udp port 111) + iptables: + chain: INPUT + in_interface: internal + protocol: udp + source: "192.168.2.0/24" + destination_port: "111" + ctstate: NEW + jump: ACCEPT + +- name: allow NFS traffic (udp port 2049) + iptables: + chain: INPUT + in_interface: internal + protocol: udp + source: "192.168.2.0/24" + destination_port: "2049" + ctstate: NEW + jump: ACCEPT + +- name: allow NFS mountd traffic (udp port 33599) + iptables: + chain: INPUT + in_interface: internal + protocol: udp + source: "192.168.2.0/24" + destination_port: "33599" + ctstate: NEW + jump: ACCEPT + +- name: allow NFS mountd traffic (tcp port 33599) + iptables: + chain: INPUT + in_interface: internal + protocol: tcp + source: "192.168.2.0/24" + destination_port: "33599" + ctstate: NEW + syn: match + jump: ACCEPT + +- name: persist iptables config + shell: "iptables-save > /etc/iptables/rules.v4" diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index bc5ce0e..ca78d8b 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -131,7 +131,7 @@ def gen_wireguard_internal( public_key=public_key, cidr=get_arkime_cidr(arkime), peers=arkime_peer_list.copy(), - listen_port=None, + listen_port=WG_LISTEN_PORT_INTERNAL, ) arkime_peer = Peer( public_key=public_key, diff --git a/terraform/bambiarkime.tf b/terraform/bambiarkime.tf index f8380f5..e8d339b 100644 --- a/terraform/bambiarkime.tf +++ b/terraform/bambiarkime.tf @@ -33,7 +33,7 @@ resource "hcloud_floating_ip_assignment" "bambiarkime_ipa" { server_id = hcloud_server.bambiarkime[count.index].id } -resource "hetznerdns_record" "bambarkime_dns" { +resource "hetznerdns_record" "bambirkime_dns" { count = var.hetznerdns_zone != null && var.use_arkime ? var.router_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "arkime${count.index + 1}${local.subdomain}" diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl index f968bb8..0599914 100644 --- a/terraform/user_data_arkime.tftpl +++ b/terraform/user_data_arkime.tftpl @@ -11,9 +11,6 @@ network: EOF netplan apply -# TODO make this reboot-safe -iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT - # Network mv "/etc/wireguard/arkime${id}.conf" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" @@ -28,6 +25,10 @@ sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" %{ endfor ~} systemctl enable --now "wg-quick@internal" +# NFS +sed -i -e "s#\[\[ARKIME_ID\]\]#${id}#g" "/etc/auto.nfs" +systemctl enable --now autofs + # Start elasticsearch cluster sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml sed -i -e "s#\[\[INITIAL_MASTER_NODES\]\]#${masters}#g" /services/BambiArkimeElasticsearch/docker-compose.yml @@ -41,9 +42,9 @@ sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/docker-entr cd /services/BambiArkime/ docker compose build -while ! curl -sq "http://192.168.2.${id}:9200"; do - echo "Waiting for elastic search to start..."; - sleep 3; +while ! curl -sq "http://192.168.2.${id}:9200/_cat/health"; do + echo "Waiting for elastic search to start... TODO: wait until green!"; + sleep 3; done if [ ${id} -eq 1 ] From e31f1ac232a03f3334d832de47705b44d31ae850 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 15 Dec 2023 21:46:43 +0100 Subject: [PATCH 23/61] off to the routers you go --- .gitignore | 1 + ansible/bambiarkime.yml | 26 -------- ansible/bambirouter.yml | 1 + .../bambi-arkime/files/arkime-capture.sh | 2 +- ansible/roles/bambi-arkime/files/config.ini | 4 +- .../files/docker-compose.elasticsearch.yml | 8 +-- .../bambi-arkime/files/docker-entrypoint.sh | 4 +- ansible/roles/bambi-arkime/tasks/main.yml | 19 +----- configgen/configgen/__init__.py | 5 +- configgen/configgen/gen_wireguard_internal.py | 49 +------------- configgen/configgen/util.py | 9 --- terraform/bambiarkime.tf | 65 ------------------- terraform/bambirouter.tf | 6 ++ terraform/user_data_arkime.tftpl | 62 ------------------ terraform/user_data_router.tftpl | 33 ++++++++++ 15 files changed, 53 insertions(+), 241 deletions(-) delete mode 100644 ansible/bambiarkime.yml delete mode 100644 terraform/bambiarkime.tf delete mode 100644 terraform/user_data_arkime.tftpl diff --git a/.gitignore b/.gitignore index e656412..d50bcdb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ config*.yml *.tfvars .terraform *.tfstate +*.tfstate.lock.info *.backup .vagrant .retry diff --git a/ansible/bambiarkime.yml b/ansible/bambiarkime.yml deleted file mode 100644 index ff02487..0000000 --- a/ansible/bambiarkime.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - roles: - - bambi-ssh-keys - - firewall - - bambi-arkime - - wireguard - - wireguard_arkime - - docker - - role: journalbeat - vars: - elk: 192.168.3.0 - - role: metricbeat - vars: - elk: 192.168.3.0 - - docker-block-external - - role: programs - vars: - program_list: - - "tmux" - - "git" diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index 867842b..33675f6 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -25,3 +25,4 @@ - "tmux" - "git" - teamvpn-configs + - bambi-arkime diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/bambi-arkime/files/arkime-capture.sh index e32c912..33ba5f3 100755 --- a/ansible/roles/bambi-arkime/files/arkime-capture.sh +++ b/ansible/roles/bambi-arkime/files/arkime-capture.sh @@ -2,6 +2,6 @@ cd /opt/arkime while : do echo "Starting Arkime capture" - /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /pcaps/pcaps -m --host $1 -n $1 + /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /pcaps -m --host $1 -n $1 sleep 5 done diff --git a/ansible/roles/bambi-arkime/files/config.ini b/ansible/roles/bambi-arkime/files/config.ini index 2649431..1dd670e 100644 --- a/ansible/roles/bambi-arkime/files/config.ini +++ b/ansible/roles/bambi-arkime/files/config.ini @@ -1,6 +1,6 @@ [default] authMode=anonymous -elasticsearch=http://[[ARKIME]]:9200 +elasticsearch=http://[[ROUTER]]:9200 rotateIndex=daily # Uncomment if this node should process the cron queries and packet search jobs, only ONE node should # process cron queries and packet search jobs @@ -11,7 +11,7 @@ httpRealm=Moloch # Semicolon ';' seperated list of interfaces to listen on for traffic interface=eth1 # The directory to save raw pcap files to -pcapDir=/pcaps/pcaps +pcapDir=/pcaps # The max raw pcap file size in gigabytes, with a max value of 36G. # The disk should have room for at least 10*maxFileSizeG diff --git a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml index f7221d7..460c82c 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml @@ -4,13 +4,13 @@ services: image: elasticsearch:8.11.1 environment: - xpack.security.enabled=false - - node.name=[[ARKIME]] + - node.name=[[ROUTER]] - cluster.name=bambiarkime - cluster.initial_master_nodes=[[INITIAL_MASTER_NODES]] - discovery.seed_hosts=[[SEED_HOSTS]] - - network.publish_host=[[ARKIME]] - - http.publish_host=[[ARKIME]] - - transport.publish_host=[[ARKIME]] + - network.publish_host=[[ROUTER]] + - http.publish_host=[[ROUTER]] + - transport.publish_host=[[ROUTER]] ports: - "9200:9200" - "9300:9300" diff --git a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh index c6a1e00..a9160f2 100755 --- a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh +++ b/ansible/roles/bambi-arkime/files/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -sh ./arkime-capture.sh [[ARKIME]] & -sh ./arkime-viewer.sh [[ARKIME]] & +sh ./arkime-capture.sh [[ROUTER]] & +sh ./arkime-viewer.sh [[ROUTER]] & wait diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml index 8bf3d10..e51229d 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -73,25 +73,8 @@ dest: /services/BambiArkime/docker-entrypoint.sh mode: '755' +# TODO once wait for elasticsearch green works # - name: build # shell: "docker compose build" # args: # chdir: /services/BambiArkime - -- name: "install autofs and nfs-client" - apt: - name: - - autofs - - nfs-client - -- name: "create /etc/auto.nfs" - shell: echo "pcaps 192.168.0.[[ARKIME_ID]]:/pcaps" >> /etc/auto.nfs - -- lineinfile: - path: /etc/auto.master - line: "/pcaps /etc/auto.nfs" - -- name: "Create /pcaps directory" - file: - path: /pcaps - state: directory diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index b23b1ac..f34a958 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -20,19 +20,17 @@ def main() -> None: parser.add_argument("--dns", type=str, default=None) parser.add_argument("--routers", type=int, default=2) parser.add_argument("--checkers", type=int, default=10) - parser.add_argument("--arkimes", type=int, default=2) args = parser.parse_args() teams: int = args.teams dns: Optional[str] = args.dns routers: int = args.routers checkers: int = args.checkers - arkimes: int = args.arkimes logger.info( f"Generating for {teams} teams, {routers} routers and {checkers} checkers" ) prepare_directories(teams) gen_wireguard_game(teams, dns, routers) - gen_wireguard_internal(teams, checkers, routers, arkimes) + gen_wireguard_internal(teams, checkers, routers) gen_passwords(teams) if dns: gen_userdata_portal(teams) @@ -46,7 +44,6 @@ def prepare_directories(teams: int) -> None: for team in range(1, teams + 1): Path(f"{DATA_DIR}/openvpn/team{team}/").mkdir(parents=True, exist_ok=True) shutil.rmtree(DATA_DIR / "export", ignore_errors=True) - Path(f"{DATA_DIR}/export/ansible/arkimes").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/routers/openvpn").mkdir( parents=True, exist_ok=True diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index ca78d8b..b1caf20 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -7,12 +7,10 @@ DATA_DIR, WG_LISTEN_PORT_INTERNAL, Peer, - WireguardArkimeConfig, WireguardCheckerConfig, WireguardConfig, WireguardRouterConfig, create_config_file, - get_arkime_cidr, get_checker_cidr, get_router_cidr_internal, get_router_index, @@ -24,7 +22,7 @@ def gen_wireguard_internal( - teams: int, checkers: int, routers: int, arkimes: int + teams: int, checkers: int, routers: int ) -> None: """ Generates the internal wireguard config files as needed, reusing keys (if present) @@ -61,11 +59,9 @@ def gen_wireguard_internal( elk_config.peers.append(engine_peer) engine_config.peers.append(elk_peer) - arkime_configs: list[WireguardArkimeConfig] = [] router_configs: list[WireguardRouterConfig] = [] checker_configs: list[WireguardCheckerConfig] = [] checker_peer_list: list[Peer] = [elk_peer, engine_peer] - arkime_peer_list: list[Peer] = [] for router_id in range(1, routers + 1): local_cidr = get_router_cidr_internal(router_id) private_key, public_key = get_wg(Path(f"wg_internal/router{router_id}.key")) @@ -97,7 +93,6 @@ def gen_wireguard_internal( elk_config.peers.append(peer) engine_config.peers.append(peer) checker_peer_list.append(peer) - arkime_peer_list.append(peer) for checker in range(1, checkers + 1): private_key, public_key = get_wg(Path(f"wg_internal/checker{checker}.key")) @@ -122,43 +117,6 @@ def gen_wireguard_internal( router_config.peers.append(checker_peer) checker_configs.append(checker_config) - for arkime in range(1, arkimes + 1): - private_key, public_key = get_wg(Path(f"wg_internal/arkime{arkime}.key")) - arkime_cidr = get_arkime_cidr(arkime) - arkime_config = WireguardArkimeConfig( - arkime_id=arkime, - private_key=private_key, - public_key=public_key, - cidr=get_arkime_cidr(arkime), - peers=arkime_peer_list.copy(), - listen_port=WG_LISTEN_PORT_INTERNAL, - ) - arkime_peer = Peer( - public_key=public_key, - allowed_ips=[arkime_cidr], - endpoint=None, - comment=f"arkime{arkime}", - ) - for router_config in router_configs: - router_config.peers.append(arkime_peer) - arkime_configs.append(arkime_config) - - # Add arkime peering (for elasticseach traffic) - for i in range(arkimes): - for j in range(arkimes): - if i == j: - continue - arkime_i = arkime_configs[i] - arkime_j = arkime_configs[j] - arkime_i.peers.append( - Peer( - public_key=arkime_j.public_key, - allowed_ips=[arkime_j.cidr], - endpoint=f"[[ARKIME_ADDRESS_{arkime_j.arkime_id}]]:{WG_LISTEN_PORT_INTERNAL}", - comment=f"arkime{arkime_j.arkime_id}", - ) - ) - # Save all to disk for router_config in router_configs: Path( @@ -170,11 +128,6 @@ def gen_wireguard_internal( f"{DATA_DIR}/export/ansible/checkers/checker{checker_config.checker_id}.conf" ).write_text(create_config_file(checker_config)) - for arkime_config in arkime_configs: - Path( - f"{DATA_DIR}/export/ansible/arkimes/arkime{arkime_config.arkime_id}.conf" - ).write_text(create_config_file(arkime_config)) - Path(f"{DATA_DIR}/export/ansible/engine.conf").write_text( create_config_file(engine_config) ) diff --git a/configgen/configgen/util.py b/configgen/configgen/util.py index e9812cb..0d16f79 100644 --- a/configgen/configgen/util.py +++ b/configgen/configgen/util.py @@ -33,11 +33,6 @@ class WireguardConfig: listen_port: Optional[int] -@dataclass -class WireguardArkimeConfig(WireguardConfig): - arkime_id: int - - @dataclass class WireguardRouterConfig(WireguardConfig): router_id: int @@ -153,10 +148,6 @@ def get_checker_cidr(checker_id: int) -> str: return f"192.168.1.{checker_id}/32" -def get_arkime_cidr(arkime_id: int) -> str: - return f"192.168.2.{arkime_id}/32" - - def get_router_cidr_game(router_id: int) -> str: return f"10.13.0.{router_id}/32" diff --git a/terraform/bambiarkime.tf b/terraform/bambiarkime.tf deleted file mode 100644 index e8d339b..0000000 --- a/terraform/bambiarkime.tf +++ /dev/null @@ -1,65 +0,0 @@ -variable "arkime_type" { - type = string - default = "cpx31" - nullable = false -} - -variable "use_arkime" { - type = bool - default = false - nullable = false -} - -locals { - subnet = "192.168.2.0/24" -} - -data "hcloud_image" "bambiarkime" { - with_selector = var.use_arkime ? "type=bambiarkime" : null - name = var.use_arkime ? null : "debian-10" - most_recent = true -} - -resource "hcloud_floating_ip" "bambiarkime_ip" { - count = var.use_arkime ? var.router_count : 0 - type = "ipv4" - name = "arkime${count.index + 1}" - home_location = var.home_location -} - -resource "hcloud_floating_ip_assignment" "bambiarkime_ipa" { - count = var.use_arkime ? var.router_count : 0 - floating_ip_id = hcloud_floating_ip.bambiarkime_ip[count.index].id - server_id = hcloud_server.bambiarkime[count.index].id -} - -resource "hetznerdns_record" "bambirkime_dns" { - count = var.hetznerdns_zone != null && var.use_arkime ? var.router_count : 0 - zone_id = data.hetznerdns_zone.zone[0].id - name = "arkime${count.index + 1}${local.subdomain}" - value = hcloud_server.bambiarkime[count.index].ipv4_address - type = "A" - ttl = 60 -} - -resource "hcloud_server" "bambiarkime" { - count = var.use_arkime ? var.router_count : 0 - name = "arkime${count.index + 1}" - image = data.hcloud_image.bambiarkime.id - location = var.home_location - server_type = var.arkime_type - ssh_keys = data.hcloud_ssh_keys.all_keys.*.id - - user_data = templatefile( - "user_data_arkime.tftpl", { - index = count.index, - id = "${count.index + 1}", - masters = join(",", [for i in range(var.router_count) : cidrhost(local.subnet, i+1)]), - seeds = join(",", setsubtract([for i in range(var.router_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) - router_ips = hcloud_floating_ip.bambirouter_ip, - elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", - engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", - arkime_ips = hcloud_floating_ip.bambiarkime_ip, - } - ) -} diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf index 6d575e5..7448dab 100644 --- a/terraform/bambirouter.tf +++ b/terraform/bambirouter.tf @@ -10,6 +10,10 @@ variable "router_count" { nullable = false } +locals { + subnet = "10.13.0.0/24" +} + data "hcloud_image" "bambirouter" { with_selector = var.router_count > 0 ? "type=bambirouter" : null name = var.router_count > 0 ? null : "debian-10" @@ -50,6 +54,8 @@ resource "hcloud_server" "bambirouter" { "user_data_router.tftpl", { index = count.index, id = "${count.index + 1}", + masters = join(",", [for i in range(var.router_count) : cidrhost(local.subnet, i+1)]), + seeds = join(",", setsubtract([for i in range(var.router_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) router_ips = hcloud_floating_ip.bambirouter_ip, elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", diff --git a/terraform/user_data_arkime.tftpl b/terraform/user_data_arkime.tftpl deleted file mode 100644 index 0599914..0000000 --- a/terraform/user_data_arkime.tftpl +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -e - -cat > /etc/netplan/60-floating-ip.yaml <<EOF -network: - version: 2 - ethernets: - eth0: - addresses: - - ${arkime_ips[index].ip_address}/32 -EOF -netplan apply - -# Network -mv "/etc/wireguard/arkime${id}.conf" "/etc/wireguard/internal.conf" -sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" -sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" -%{ for router_index, router in router_ips ~} - sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" - sed -i -e "s#\[\[ROUTER_ADDRESS_${router_index+1}\]\]#${router.ip_address}#g" "/etc/wireguard/internal.conf" -%{ endfor ~} -%{ for arkime_index, arkime in arkime_ips ~} - sed -i -e "s#\[\[ARKIME_ADDRESS_${arkime_index+1}\]\]#${arkime.ip_address}#g" "/etc/wireguard/internal.conf" - sed -i -e "s#\[\[ARKIME_ADDRESS_${arkime_index+1}\]\]#${arkime.ip_address}#g" "/etc/wireguard/internal.conf" -%{ endfor ~} -systemctl enable --now "wg-quick@internal" - -# NFS -sed -i -e "s#\[\[ARKIME_ID\]\]#${id}#g" "/etc/auto.nfs" -systemctl enable --now autofs - -# Start elasticsearch cluster -sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml -sed -i -e "s#\[\[INITIAL_MASTER_NODES\]\]#${masters}#g" /services/BambiArkimeElasticsearch/docker-compose.yml -sed -i -e "s#\[\[SEED_HOSTS\]\]#${seeds}#g" /services/BambiArkimeElasticsearch/docker-compose.yml -cd /services/BambiArkimeElasticsearch/ -docker compose up -d - -# Start Arkime cluster -sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/config.ini -sed -i -e "s#\[\[ARKIME\]\]#192.168.2.${id}#g" /services/BambiArkime/docker-entrypoint.sh -cd /services/BambiArkime/ -docker compose build - -while ! curl -sq "http://192.168.2.${id}:9200/_cat/health"; do - echo "Waiting for elastic search to start... TODO: wait until green!"; - sleep 3; -done - -if [ ${id} -eq 1 ] -then - docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://192.168.2.${id}:9200" init - docker compose run --entrypoint="/opt/arkime/bin/arkime_add_user.sh" arkime admin "Admin" admin --admin - curl -X PUT "http://192.168.2.${id}:9200/bambi-arkime-init?pretty" -fi - -while ! curl -s --head --show-error --fail "http://192.168.2.${id}:9200/bambi-arkime-init/"; do - echo "Waiting for marker index to exist..."; - sleep 3; -done - -docker compose up -d diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 9d76645..87fbd7c 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -28,3 +28,36 @@ sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" systemctl enable --now "wg-quick@internal" systemctl enable --now "wg-quick@router" + +# Arkime +# Start elasticsearch cluster +sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +sed -i -e "s#\[\[INITIAL_MASTER_NODES\]\]#${masters}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +sed -i -e "s#\[\[SEED_HOSTS\]\]#${seeds}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +cd /services/BambiArkimeElasticsearch/ +docker compose up -d + +# Start Arkime cluster +sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkime/config.ini +sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkime/docker-entrypoint.sh +cd /services/BambiArkime/ +docker compose build + +while ! curl -sq "http://10.13.0.${id}:9200/_cat/health"; do + echo "Waiting for elastic search to start... TODO: wait until green!"; + sleep 3; +done + +if [ ${id} -eq 1 ] +then + docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://10.13.0.${id}:9200" init + docker compose run --entrypoint="/opt/arkime/bin/arkime_add_user.sh" arkime admin "Admin" admin --admin + curl -X PUT "http://10.13.0.${id}:9200/bambi-arkime-init?pretty" +fi + +while ! curl -s --head --show-error --fail "http://10.13.0.${id}:9200/bambi-arkime-init/"; do + echo "Waiting for marker index to exist..."; + sleep 3; +done + +docker compose up -d From 28294462b9a9697aef60a41924a165fafb437d2f Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 16 Dec 2023 12:28:47 +0100 Subject: [PATCH 24/61] arkime nightly works --- ansible/roles/bambi-arkime/files/Dockerfile | 2 +- ansible/roles/router_trafficcapture/files/enorouterdump.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/bambi-arkime/files/Dockerfile b/ansible/roles/bambi-arkime/files/Dockerfile index f24b593..5ff683a 100644 --- a/ansible/roles/bambi-arkime/files/Dockerfile +++ b/ansible/roles/bambi-arkime/files/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.04 RUN apt-get update && apt-get install -y libwww-perl libjson-perl ethtool libyaml-dev liblua5.4-0 libmaxminddb0 libcurl4 libpcap0.8 libglib2.0-0 libnghttp2-14 libyara8 librdkafka1 curl WORKDIR /BambiArkime -ADD https://github.com/arkime/arkime/releases/download/v5.0.0-rc1/arkime_5.0.0-rc1-1.ubuntu2204_amd64.deb . +ADD https://github.com/arkime/arkime/releases/download/last-commit/arkime-main_ubuntu2204_amd64.deb . RUN dpkg -i ./*.deb; apt-get install -fy RUN touch /opt/arkime/etc/oui.txt diff --git a/ansible/roles/router_trafficcapture/files/enorouterdump.service b/ansible/roles/router_trafficcapture/files/enorouterdump.service index 5f3f5aa..3ec4f7e 100644 --- a/ansible/roles/router_trafficcapture/files/enorouterdump.service +++ b/ansible/roles/router_trafficcapture/files/enorouterdump.service @@ -8,7 +8,7 @@ RestartSec=1 TimeoutStartSec=300 WorkingDirectory=/pcaps -ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 +ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 -z touch src net 10.0.0.0/16 and dst net 10.0.0.0/16 [Install] WantedBy=multi-user.target From 54b6a42cb9743a379bd18858cc24a9a3abbebc7a Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Fri, 5 Apr 2024 21:05:52 +0200 Subject: [PATCH 25/61] refactor playbooks and roles to declare programs in depdendency --- ansible/bambichecker.yml | 6 +----- ansible/bambielk.yml | 6 +----- ansible/bambiengine.yml | 6 +----- ansible/bambirouter.yml | 6 +----- ansible/bambivulnbox.yml | 4 +--- ansible/roles/enoelk/meta/main.yml | 4 ++++ ansible/roles/enoengine/meta/main.yml | 4 ++++ ansible/roles/pip-packages/meta/main.yml | 5 +++++ ansible/roles/programs/tasks/main.yml | 5 ----- ansible/roles/{programs => tmux}/files/.tmux.conf | 0 ansible/roles/tmux/meta/main.yml | 5 +++++ ansible/roles/tmux/tasks/main.yml | 5 +++++ 12 files changed, 28 insertions(+), 28 deletions(-) create mode 100644 ansible/roles/pip-packages/meta/main.yml rename ansible/roles/{programs => tmux}/files/.tmux.conf (100%) create mode 100644 ansible/roles/tmux/meta/main.yml create mode 100644 ansible/roles/tmux/tasks/main.yml diff --git a/ansible/bambichecker.yml b/ansible/bambichecker.yml index 8846186..728b3aa 100644 --- a/ansible/bambichecker.yml +++ b/ansible/bambichecker.yml @@ -16,8 +16,4 @@ - metricbeat - vuln_checkers - docker-block-external - - role: programs - vars: - program_list: - - "tmux" - - "git" + - tmux diff --git a/ansible/bambielk.yml b/ansible/bambielk.yml index 6e06329..1b64fb8 100644 --- a/ansible/bambielk.yml +++ b/ansible/bambielk.yml @@ -16,8 +16,4 @@ - filebeat - journalbeat - metricbeat - - role: programs - vars: - program_list: - - "tmux" - - "git" + - tmux diff --git a/ansible/bambiengine.yml b/ansible/bambiengine.yml index a45f95d..3cf6567 100644 --- a/ansible/bambiengine.yml +++ b/ansible/bambiengine.yml @@ -16,8 +16,4 @@ - journalbeat - metricbeat - docker-block-external - - role: programs - vars: - program_list: - - "tmux" - - "git" + - tmux diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index e377c56..43df97e 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -14,10 +14,6 @@ - firewall - bambi-wireguard-router - router_trafficcapture - - role: programs - vars: - program_list: - - "tmux" - - "git" + - tmux - teamvpn-configs - bambi-arkime diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index 412857c..7e761a4 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -15,13 +15,11 @@ - wireguard - enomoloch - bambixploit + - tmux - role: programs vars: program_list: - - tmux - unzip - - git - - python3-pip - role: pip-packages vars: pip_list: diff --git a/ansible/roles/enoelk/meta/main.yml b/ansible/roles/enoelk/meta/main.yml index c1f68fc..18976ce 100644 --- a/ansible/roles/enoelk/meta/main.yml +++ b/ansible/roles/enoelk/meta/main.yml @@ -1,2 +1,6 @@ dependencies: - role: "docker" +- role: "programs" + vars: + program_list: + - git diff --git a/ansible/roles/enoengine/meta/main.yml b/ansible/roles/enoengine/meta/main.yml index 03efa44..8d209af 100644 --- a/ansible/roles/enoengine/meta/main.yml +++ b/ansible/roles/enoengine/meta/main.yml @@ -1,3 +1,7 @@ dependencies: - role: "dotnetsdk" - role: "docker" +- role: "programs" + vars: + program_list: + - git diff --git a/ansible/roles/pip-packages/meta/main.yml b/ansible/roles/pip-packages/meta/main.yml new file mode 100644 index 0000000..87bbb4c --- /dev/null +++ b/ansible/roles/pip-packages/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: +- role: "programs" + vars: + program_list: + - python3-pip diff --git a/ansible/roles/programs/tasks/main.yml b/ansible/roles/programs/tasks/main.yml index 6ccd4e5..53dedf4 100644 --- a/ansible/roles/programs/tasks/main.yml +++ b/ansible/roles/programs/tasks/main.yml @@ -6,8 +6,3 @@ autoremove: yes cache_valid_time: 3000 state: present - - - name: Copy tmux conf - copy: - src: .tmux.conf - dest: /root/.tmux.conf diff --git a/ansible/roles/programs/files/.tmux.conf b/ansible/roles/tmux/files/.tmux.conf similarity index 100% rename from ansible/roles/programs/files/.tmux.conf rename to ansible/roles/tmux/files/.tmux.conf diff --git a/ansible/roles/tmux/meta/main.yml b/ansible/roles/tmux/meta/main.yml new file mode 100644 index 0000000..1ed982e --- /dev/null +++ b/ansible/roles/tmux/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: +- role: "programs" + vars: + program_list: + - tmux diff --git a/ansible/roles/tmux/tasks/main.yml b/ansible/roles/tmux/tasks/main.yml new file mode 100644 index 0000000..892f90e --- /dev/null +++ b/ansible/roles/tmux/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Copy tmux conf + copy: + src: .tmux.conf + dest: /root/.tmux.conf From 1fdab054e355a9a10dc45d98066e2208808ea2bd Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Fri, 5 Apr 2024 21:52:53 +0200 Subject: [PATCH 26/61] ansible role refactoring --- ansible/bambichecker.yml | 9 ++++----- ansible/bambielk.yml | 9 ++++----- ansible/bambiengine.yml | 9 ++++----- ansible/bambirouter.yml | 4 +--- ansible/bambivulnbox.yml | 5 +---- ansible/roles/bambi-arkime/tasks/main.yml | 3 ++- ansible/roles/bambi-wireguard-router/meta/main.yml | 4 ++++ ansible/roles/bambi-wireguard-router/tasks/main.yml | 3 ++- ansible/roles/base/meta/main.yml | 5 +++++ ansible/roles/base/tasks/main.yml | 5 +++++ ansible/roles/docker-block-external/tasks/main.yml | 3 ++- ansible/roles/enoelk/tasks/main.yml | 3 ++- ansible/roles/enoengine/tasks/main.yml | 3 ++- ansible/roles/enomoloch/tasks/main.yml | 3 ++- ansible/roles/firewall/meta/main.yml | 2 -- ansible/roles/firewall/tasks/main.yml | 8 +++----- ansible/roles/iptables-persistent/meta/main.yml | 6 ++++++ ansible/roles/iptables-persistent/tasks/main.yml | 10 +++++----- ansible/roles/pip-packages/tasks/main.yml | 8 -------- ansible/roles/router_trafficcapture/tasks/main.yml | 3 ++- ansible/roles/teamvpn-configs/meta/main.yml | 4 ++++ ansible/roles/teamvpn-configs/tasks/main.yml | 8 ++------ ansible/roles/wireguard/meta/main.yml | 5 +++++ ansible/roles/wireguard/tasks/main.yml | 10 ---------- ansible/roles/wireguard_arkime/tasks/main.yml | 5 ----- ansible/roles/wireguard_checker/tasks/main.yml | 5 ----- ansible/roles/wireguard_configs/meta/main.yml | 2 ++ ansible/roles/wireguard_configs/tasks/main.yml | 5 +++++ ansible/roles/wireguard_elk/tasks/main.yml | 5 ----- ansible/roles/wireguard_engine/tasks/main.yml | 5 ----- configgen/configgen/gen_wireguard_internal.py | 4 ++-- 31 files changed, 76 insertions(+), 87 deletions(-) create mode 100644 ansible/roles/base/meta/main.yml create mode 100644 ansible/roles/base/tasks/main.yml delete mode 100644 ansible/roles/firewall/meta/main.yml create mode 100644 ansible/roles/iptables-persistent/meta/main.yml create mode 100644 ansible/roles/teamvpn-configs/meta/main.yml create mode 100644 ansible/roles/wireguard/meta/main.yml delete mode 100644 ansible/roles/wireguard_arkime/tasks/main.yml delete mode 100644 ansible/roles/wireguard_checker/tasks/main.yml create mode 100644 ansible/roles/wireguard_configs/meta/main.yml create mode 100644 ansible/roles/wireguard_configs/tasks/main.yml delete mode 100644 ansible/roles/wireguard_elk/tasks/main.yml delete mode 100644 ansible/roles/wireguard_engine/tasks/main.yml diff --git a/ansible/bambichecker.yml b/ansible/bambichecker.yml index 728b3aa..fc3e87c 100644 --- a/ansible/bambichecker.yml +++ b/ansible/bambichecker.yml @@ -7,13 +7,12 @@ - static.yml roles: - - bambi-ssh-keys - - firewall - - wireguard - - wireguard_checker + - base + - role: wireguard_configs + vars: + config_dir: checkers - filebeat - journalbeat - metricbeat - vuln_checkers - docker-block-external - - tmux diff --git a/ansible/bambielk.yml b/ansible/bambielk.yml index 1b64fb8..90d6ad7 100644 --- a/ansible/bambielk.yml +++ b/ansible/bambielk.yml @@ -7,13 +7,12 @@ - static.yml roles: - - bambi-ssh-keys - - firewall - - wireguard + - base - enoelk - docker-block-external - - wireguard_elk + - role: wireguard_configs + vars: + config_dir: elk - filebeat - journalbeat - metricbeat - - tmux diff --git a/ansible/bambiengine.yml b/ansible/bambiengine.yml index 3cf6567..bc6c54b 100644 --- a/ansible/bambiengine.yml +++ b/ansible/bambiengine.yml @@ -7,13 +7,12 @@ - static.yml roles: - - bambi-ssh-keys - - firewall - - wireguard + - base - enoengine - - wireguard_engine + - role: wireguard_configs + vars: + config_dir: engine - filebeat - journalbeat - metricbeat - docker-block-external - - tmux diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index 43df97e..c10b7aa 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -7,13 +7,11 @@ - static.yml roles: + - base - filebeat - journalbeat - metricbeat - - bambi-ssh-keys - - firewall - bambi-wireguard-router - router_trafficcapture - - tmux - teamvpn-configs - bambi-arkime diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index 7e761a4..4bb8243 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -7,15 +7,12 @@ - static.yml roles: + - base - docker - vuln_services - - bambi-ssh-keys - - firewall - docker-block-external - - wireguard - enomoloch - bambixploit - - tmux - role: programs vars: program_list: diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml index e51229d..c6a9a0b 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -12,7 +12,8 @@ - shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 9300 -j ACCEPT - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" - name: Increase max_map_count sysctl: diff --git a/ansible/roles/bambi-wireguard-router/meta/main.yml b/ansible/roles/bambi-wireguard-router/meta/main.yml index 05473c3..28fe50f 100644 --- a/ansible/roles/bambi-wireguard-router/meta/main.yml +++ b/ansible/roles/bambi-wireguard-router/meta/main.yml @@ -1,3 +1,7 @@ dependencies: - role: "wireguard" - role: "firewall" +- role: "programs" + vars: + program_list: + - iptables-persistent diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml index 5c6d519..60f1715 100644 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ b/ansible/roles/bambi-wireguard-router/tasks/main.yml @@ -135,4 +135,5 @@ with_sequence: start=1 end=255 - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/base/meta/main.yml b/ansible/roles/base/meta/main.yml new file mode 100644 index 0000000..2710153 --- /dev/null +++ b/ansible/roles/base/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: +- role: "bambi-ssh-keys" +- role: "firewall" +- role: "tmux" +- role: "wireguard" diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml new file mode 100644 index 0000000..4384fea --- /dev/null +++ b/ansible/roles/base/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Update and upgrade apt packages + apt: + upgrade: "yes" + update_cache: yes diff --git a/ansible/roles/docker-block-external/tasks/main.yml b/ansible/roles/docker-block-external/tasks/main.yml index fb9ca9e..f5ed6eb 100644 --- a/ansible/roles/docker-block-external/tasks/main.yml +++ b/ansible/roles/docker-block-external/tasks/main.yml @@ -20,4 +20,5 @@ state: absent - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/enoelk/tasks/main.yml b/ansible/roles/enoelk/tasks/main.yml index 19f59e3..1597874 100644 --- a/ansible/roles/enoelk/tasks/main.yml +++ b/ansible/roles/enoelk/tasks/main.yml @@ -29,4 +29,5 @@ jump: ACCEPT - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/enoengine/tasks/main.yml b/ansible/roles/enoengine/tasks/main.yml index fec2db0..aa525d1 100644 --- a/ansible/roles/enoengine/tasks/main.yml +++ b/ansible/roles/enoengine/tasks/main.yml @@ -60,7 +60,8 @@ jump: ACCEPT - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" - name: Create /services/data ansible.builtin.file: diff --git a/ansible/roles/enomoloch/tasks/main.yml b/ansible/roles/enomoloch/tasks/main.yml index 1d43cce..0257e1a 100644 --- a/ansible/roles/enomoloch/tasks/main.yml +++ b/ansible/roles/enomoloch/tasks/main.yml @@ -62,4 +62,5 @@ jump: DROP - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/firewall/meta/main.yml b/ansible/roles/firewall/meta/main.yml deleted file mode 100644 index 1d50143..0000000 --- a/ansible/roles/firewall/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: -- role: "iptables-persistent" diff --git a/ansible/roles/firewall/tasks/main.yml b/ansible/roles/firewall/tasks/main.yml index 6c663b9..62f89a2 100644 --- a/ansible/roles/firewall/tasks/main.yml +++ b/ansible/roles/firewall/tasks/main.yml @@ -37,9 +37,6 @@ chain: FORWARD policy: DROP - - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" - - name: allow IPv6 loopback traffic iptables: chain: INPUT @@ -103,5 +100,6 @@ policy: DROP ip_version: ipv6 - - name: persist ip6tables config - shell: "ip6tables-save > /etc/iptables/rules.v6" + - name: persist iptables config + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/iptables-persistent/meta/main.yml b/ansible/roles/iptables-persistent/meta/main.yml new file mode 100644 index 0000000..fc0e0b6 --- /dev/null +++ b/ansible/roles/iptables-persistent/meta/main.yml @@ -0,0 +1,6 @@ +allow_duplicates: true +dependencies: +- role: "programs" + vars: + program_list: + - iptables-persistent diff --git a/ansible/roles/iptables-persistent/tasks/main.yml b/ansible/roles/iptables-persistent/tasks/main.yml index b17bb3f..f7d4300 100644 --- a/ansible/roles/iptables-persistent/tasks/main.yml +++ b/ansible/roles/iptables-persistent/tasks/main.yml @@ -1,6 +1,6 @@ --- - - name: install iptables-persistent - apt: - name: iptables-persistent - state: present - update_cache: yes + - name: persist iptables config + shell: "iptables-save > /etc/iptables/rules.v4" + + - name: persist ip6tables config + shell: "ip6tables-save > /etc/iptables/rules.v6" diff --git a/ansible/roles/pip-packages/tasks/main.yml b/ansible/roles/pip-packages/tasks/main.yml index bc9bd1e..9da92b2 100644 --- a/ansible/roles/pip-packages/tasks/main.yml +++ b/ansible/roles/pip-packages/tasks/main.yml @@ -1,12 +1,4 @@ --- - - name: Install python3-pip - apt: - name: "python3-pip" - force_apt_get: yes - autoremove: yes - cache_valid_time: 3000 - state: present - - name: Install pip packages ansible.builtin.pip: name: "{{ pip_list }}" diff --git a/ansible/roles/router_trafficcapture/tasks/main.yml b/ansible/roles/router_trafficcapture/tasks/main.yml index 17bd4b4..1d3488d 100644 --- a/ansible/roles/router_trafficcapture/tasks/main.yml +++ b/ansible/roles/router_trafficcapture/tasks/main.yml @@ -121,4 +121,5 @@ jump: ACCEPT - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/teamvpn-configs/meta/main.yml b/ansible/roles/teamvpn-configs/meta/main.yml new file mode 100644 index 0000000..e475f92 --- /dev/null +++ b/ansible/roles/teamvpn-configs/meta/main.yml @@ -0,0 +1,4 @@ +- role: "programs" + vars: + program_list: + - openvpn diff --git a/ansible/roles/teamvpn-configs/tasks/main.yml b/ansible/roles/teamvpn-configs/tasks/main.yml index 57e2395..1623a57 100644 --- a/ansible/roles/teamvpn-configs/tasks/main.yml +++ b/ansible/roles/teamvpn-configs/tasks/main.yml @@ -1,9 +1,4 @@ --- -- name: "Install openvpn" - apt: - package: - - openvpn - - name: copy zipped team VPN configs copy: src: "{{ playbook_dir }}/../config/export/ansible/routers/openvpn/" @@ -18,4 +13,5 @@ jump: ACCEPT - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" + include_role: + name: "iptables-persistent" diff --git a/ansible/roles/wireguard/meta/main.yml b/ansible/roles/wireguard/meta/main.yml new file mode 100644 index 0000000..161b64b --- /dev/null +++ b/ansible/roles/wireguard/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: +- role: "programs" + vars: + program_list: + - wireguard diff --git a/ansible/roles/wireguard/tasks/main.yml b/ansible/roles/wireguard/tasks/main.yml index c11e7ff..667caae 100644 --- a/ansible/roles/wireguard/tasks/main.yml +++ b/ansible/roles/wireguard/tasks/main.yml @@ -1,14 +1,4 @@ --- - - name: Update and upgrade apt packages - apt: - upgrade: "yes" - update_cache: yes - - - name: Install wg - apt: - name: wireguard - state: present - - name: Ensure wg directory exists file: path: /etc/wireguard diff --git a/ansible/roles/wireguard_arkime/tasks/main.yml b/ansible/roles/wireguard_arkime/tasks/main.yml deleted file mode 100644 index 87246fb..0000000 --- a/ansible/roles/wireguard_arkime/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - - name: Copy wg configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/arkimes/" - dest: /etc/wireguard/ diff --git a/ansible/roles/wireguard_checker/tasks/main.yml b/ansible/roles/wireguard_checker/tasks/main.yml deleted file mode 100644 index b14e21e..0000000 --- a/ansible/roles/wireguard_checker/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - - name: Copy wg configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/checkers/" - dest: /etc/wireguard/ diff --git a/ansible/roles/wireguard_configs/meta/main.yml b/ansible/roles/wireguard_configs/meta/main.yml new file mode 100644 index 0000000..656bcf7 --- /dev/null +++ b/ansible/roles/wireguard_configs/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: +- role: "wireguard" diff --git a/ansible/roles/wireguard_configs/tasks/main.yml b/ansible/roles/wireguard_configs/tasks/main.yml new file mode 100644 index 0000000..27994c4 --- /dev/null +++ b/ansible/roles/wireguard_configs/tasks/main.yml @@ -0,0 +1,5 @@ +--- + - name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/{{ config_dir }}/" + dest: /etc/wireguard/ diff --git a/ansible/roles/wireguard_elk/tasks/main.yml b/ansible/roles/wireguard_elk/tasks/main.yml deleted file mode 100644 index b6ced41..0000000 --- a/ansible/roles/wireguard_elk/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - - name: Copy wg configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/elk.conf" - dest: /etc/wireguard/internal.conf diff --git a/ansible/roles/wireguard_engine/tasks/main.yml b/ansible/roles/wireguard_engine/tasks/main.yml deleted file mode 100644 index 83d3aac..0000000 --- a/ansible/roles/wireguard_engine/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - - name: Copy wg configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/engine.conf" - dest: /etc/wireguard/internal.conf diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index b1caf20..edbb164 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -128,9 +128,9 @@ def gen_wireguard_internal( f"{DATA_DIR}/export/ansible/checkers/checker{checker_config.checker_id}.conf" ).write_text(create_config_file(checker_config)) - Path(f"{DATA_DIR}/export/ansible/engine.conf").write_text( + Path(f"{DATA_DIR}/export/ansible/engine/engine.conf").write_text( create_config_file(engine_config) ) - Path(f"{DATA_DIR}/export/ansible/elk.conf").write_text( + Path(f"{DATA_DIR}/export/ansible/elk/elk.conf").write_text( create_config_file(elk_config) ) From e07a5f4fcc053ec29570677eec4ff8ed3a0bee0b Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Fri, 5 Apr 2024 22:14:20 +0200 Subject: [PATCH 27/61] create elk/engine config directories --- configgen/configgen/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index f34a958..9bbf66b 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -45,6 +45,8 @@ def prepare_directories(teams: int) -> None: Path(f"{DATA_DIR}/openvpn/team{team}/").mkdir(parents=True, exist_ok=True) shutil.rmtree(DATA_DIR / "export", ignore_errors=True) Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/export/ansible/elk").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/export/ansible/engine").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/routers/openvpn").mkdir( parents=True, exist_ok=True ) From 210717949adcf4daa8fbfdeef60eee4e53390cd9 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Fri, 5 Apr 2024 22:21:51 +0200 Subject: [PATCH 28/61] add missing dependencies: in teamvpn-configs role --- ansible/roles/teamvpn-configs/meta/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/teamvpn-configs/meta/main.yml b/ansible/roles/teamvpn-configs/meta/main.yml index e475f92..1ab1c16 100644 --- a/ansible/roles/teamvpn-configs/meta/main.yml +++ b/ansible/roles/teamvpn-configs/meta/main.yml @@ -1,3 +1,4 @@ +dependencies: - role: "programs" vars: program_list: From 4fa108cd47218f72029ac0cd82fd492715588f71 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 6 Apr 2024 12:56:29 +0200 Subject: [PATCH 29/61] Lucas told me to push. No guarantees! --- Dockerfile | 26 +++++++------------ README.md | 7 ++--- ansible/roles/bambi-arkime/files/Dockerfile | 15 ++++------- .../files/docker-compose.arkime.yml | 2 +- packer/bambiarkime.json | 26 ------------------- packer/bambichecker.json | 10 +++---- packer/bambielk.json | 10 +++---- packer/bambiengine.json | 10 +++---- packer/bambirouter.json | 10 +++---- packer/bambivulnbox.json | 10 +++---- 10 files changed, 43 insertions(+), 83 deletions(-) delete mode 100644 packer/bambiarkime.json diff --git a/Dockerfile b/Dockerfile index bb9dbdf..5ec4ee7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,19 @@ -FROM ubuntu:22.04 +FROM ubuntu:23.10 # Core deps RUN apt-get update RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata -RUN apt-get install -y --no-install-recommends rsync git less tmux python3 curl wireguard python3-pip unzip file nano dnsutils jq \ - software-properties-common gpg-agent # for ansible and packer install +RUN apt-get install -y --no-install-recommends openssh-client rsync git less tmux python3 curl wireguard unzip file nano dnsutils jq \ + software-properties-common gpg-agent pipx # for ansible and packer install -# Poetry -RUN pip install poetry && poetry config virtualenvs.in-project false +# Poetry and Ansible +RUN pipx install poetry && pipx install --include-deps ansible +ENV PATH="/root/.local/bin:${PATH}" -# Ansible -RUN add-apt-repository --yes --update ppa:ansible/ansible && apt-get install -y ansible - -# Terrorform -RUN ls -alh /usr/local/bin -RUN curl https://releases.hashicorp.com/terraform/1.6.4/terraform_1.6.4_linux_amd64.zip > terraform.zip && \ - unzip terraform.zip && \ - mv terraform /usr/local/bin/ - -# Packer +# Packer and Terraform RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - && \ apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" && \ - apt-get update && apt-get install packer && \ + apt-get update && apt-get install packer terraform && \ packer plugins install github.com/hashicorp/hcloud && \ packer plugins install github.com/hashicorp/ansible @@ -33,7 +25,7 @@ ENV PATH="/usr/share/easy-rsa:${PATH}" RUN echo "set -g mouse on" > /root/.tmux.conf # fix SSH host key checking -RUN mkdir /root/.ssh && echo "Host 127.0.0.1\n HostKeyAlgorithms=+ssh-rsa\n PubkeyAcceptedKeyTypes=+ssh-rsa" > /root/.ssh/config +# RUN mkdir /root/.ssh && echo "Host 127.0.0.1\n HostKeyAlgorithms=+ssh-rsa\n PubkeyAcceptedKeyTypes=+ssh-rsa" > /root/.ssh/config WORKDIR /bambictf diff --git a/README.md b/README.md index 0d8463c..b493fea 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,11 @@ export HCLOUD_TOKEN="..." - Run the container (`docker compose up -d`) - Invoke a bash in the container (`docker compose exec bambictf bash`) - Build configs - - `cd /bambictf/config` - - `./gen_config.sh` + - `cd /bambictf/configgen` + - `poetry install` (once) + - `poetry run configgen --teams 4 --routers 2 --dns test.bambi.ovh` - Ship everything to the EnoCTFPortal: - - `cp -r ./export/ /services/EnoCTFPortal/data/teamdata` (or whereever it is) + - `cp -r ./export/portal /services/EnoCTFPortal/data/teamdata` (or whereever it is) - Builds VMs - `cd /bambictf/packer` - `packer build bambichecker.json` diff --git a/ansible/roles/bambi-arkime/files/Dockerfile b/ansible/roles/bambi-arkime/files/Dockerfile index 5ff683a..fea70dd 100644 --- a/ansible/roles/bambi-arkime/files/Dockerfile +++ b/ansible/roles/bambi-arkime/files/Dockerfile @@ -1,11 +1,6 @@ -FROM ubuntu:22.04 -RUN apt-get update && apt-get install -y libwww-perl libjson-perl ethtool libyaml-dev liblua5.4-0 libmaxminddb0 libcurl4 libpcap0.8 libglib2.0-0 libnghttp2-14 libyara8 librdkafka1 curl +FROM ghcr.io/enoflag/enoarkime:5.1.0 -WORKDIR /BambiArkime -ADD https://github.com/arkime/arkime/releases/download/last-commit/arkime-main_ubuntu2204_amd64.deb . -RUN dpkg -i ./*.deb; apt-get install -fy - -RUN touch /opt/arkime/etc/oui.txt -RUN touch /opt/arkime/etc/ipv4-address-space.csv - -ENTRYPOINT /BambiArkime/docker-entrypoint.sh +COPY arkime-viewer.sh arkime-viewer.sh +COPY arkime-capture.sh arkime-capture.sh +COPY docker-entrypoint.sh docker-entrypoint.sh +COPY config.ini /opt/arkime/etc/config.ini diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml index c9ad6a2..a9e93fd 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml @@ -6,6 +6,6 @@ services: ports: - 8005:8005 volumes: - - "/pcaps:/pcaps" + - "/pcaps:/opt/arkime/raw" - ".:/BambiArkime:ro" - "./config.ini:/opt/arkime/etc/config.ini:ro" diff --git a/packer/bambiarkime.json b/packer/bambiarkime.json deleted file mode 100644 index bade5bc..0000000 --- a/packer/bambiarkime.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "provisioners": [ - { - "type": "ansible", - "playbook_file": "../ansible/bambiarkime.yml", - "host_alias": "packer-arkime", - "extra_arguments": [ - "--extra-vars", - "ansible_python_interpreter=/usr/bin/python3" - ] - } - ], - "builders": [ - { - "type": "hcloud", - "image": "ubuntu-22.04", - "location": "fsn1", - "server_type": "cx11", - "ssh_username": "root", - "snapshot_name": "bambiarkime-{{timestamp}}", - "snapshot_labels": { - "type": "bambiarkime" - } - } - ] -} diff --git a/packer/bambichecker.json b/packer/bambichecker.json index 9cb58b1..35afac5 100644 --- a/packer/bambichecker.json +++ b/packer/bambichecker.json @@ -4,10 +4,10 @@ "type": "ansible", "playbook_file": "../ansible/bambichecker.yml", "host_alias": "packer-checker", - "extra_arguments": [ - "--extra-vars", - "ansible_python_interpreter=/usr/bin/python3" - ] + "ansible_env_vars": [ + "ANSIBLE_PIPELINING=True" + ], + "extra_arguments": [ "--scp-extra-args", "'-O'" ] } ], "builders": [ @@ -21,7 +21,7 @@ "snapshot_labels": { "type": "bambichecker" }, - "user_data": "#!/bin/sh\necho PubkeyAcceptedKeyTypes=+ssh-rsa >> /etc/ssh/sshd_config; service ssh reload" + "temporary_key_pair_type": "ecdsa" } ] } diff --git a/packer/bambielk.json b/packer/bambielk.json index d3d57e7..21286f8 100644 --- a/packer/bambielk.json +++ b/packer/bambielk.json @@ -4,10 +4,10 @@ "type": "ansible", "playbook_file": "../ansible/bambielk.yml", "host_alias": "packer-elk", - "extra_arguments": [ - "--extra-vars", - "ansible_python_interpreter=/usr/bin/python3" - ] + "ansible_env_vars": [ + "ANSIBLE_PIPELINING=True" + ], + "extra_arguments": [ "--scp-extra-args", "'-O'" ] } ], "builders": [ @@ -21,7 +21,7 @@ "snapshot_labels": { "type": "bambielk" }, - "user_data": "#!/bin/sh\necho PubkeyAcceptedKeyTypes=+ssh-rsa >> /etc/ssh/sshd_config; service ssh reload" + "temporary_key_pair_type": "ecdsa" } ] } \ No newline at end of file diff --git a/packer/bambiengine.json b/packer/bambiengine.json index 982707b..28470a5 100644 --- a/packer/bambiengine.json +++ b/packer/bambiengine.json @@ -4,10 +4,10 @@ "type": "ansible", "playbook_file": "../ansible/bambiengine.yml", "host_alias": "packer-engine", - "extra_arguments": [ - "--extra-vars", - "ansible_python_interpreter=/usr/bin/python3" - ] + "ansible_env_vars": [ + "ANSIBLE_PIPELINING=True" + ], + "extra_arguments": [ "--scp-extra-args", "'-O'" ] } ], "builders": [ @@ -21,7 +21,7 @@ "snapshot_labels": { "type": "bambiengine" }, - "user_data": "#!/bin/sh\necho PubkeyAcceptedKeyTypes=+ssh-rsa >> /etc/ssh/sshd_config; service ssh reload" + "temporary_key_pair_type": "ecdsa" } ] } diff --git a/packer/bambirouter.json b/packer/bambirouter.json index 1d94647..58e5c26 100644 --- a/packer/bambirouter.json +++ b/packer/bambirouter.json @@ -4,13 +4,11 @@ "type": "ansible", "playbook_file": "../ansible/bambirouter.yml", "host_alias": "packer-router", - "extra_arguments": [ - "--extra-vars", - "ansible_python_interpreter=/usr/bin/python3" - ], "ansible_env_vars": [ "ANSIBLE_PIPELINING=True" - ] + ], + "extra_arguments": [ "--scp-extra-args", "'-O'" ] + } ], "builders": [ @@ -24,7 +22,7 @@ "snapshot_labels": { "type": "bambirouter" }, - "user_data": "#!/bin/sh\necho PubkeyAcceptedKeyTypes=+ssh-rsa >> /etc/ssh/sshd_config; service ssh reload" + "temporary_key_pair_type": "ecdsa" } ] } diff --git a/packer/bambivulnbox.json b/packer/bambivulnbox.json index fffcfda..85f6e23 100644 --- a/packer/bambivulnbox.json +++ b/packer/bambivulnbox.json @@ -4,10 +4,10 @@ "type": "ansible", "playbook_file": "../ansible/bambivulnbox.yml", "host_alias": "packer-vulnbox", - "extra_arguments": [ - "--extra-vars", - "ansible_python_interpreter=/usr/bin/python3" - ] + "ansible_env_vars": [ + "ANSIBLE_PIPELINING=True" + ], + "extra_arguments": [ "--scp-extra-args", "'-O'" ] } ], "builders": [ @@ -21,7 +21,7 @@ "snapshot_labels": { "type": "bambivulnbox" }, - "user_data": "#!/bin/sh\necho 'PubkeyAcceptedKeyTypes=+ssh-rsa\nPermitRootLogin yes' >> /etc/ssh/sshd_config; service ssh reload" + "temporary_key_pair_type": "ecdsa" } ] } From 6fbfaee0e4de540a5f0c32a00baac9adfb925aa9 Mon Sep 17 00:00:00 2001 From: ldruschk <14059613+ldruschk@users.noreply.github.com> Date: Sat, 6 Apr 2024 18:02:31 +0200 Subject: [PATCH 30/61] Ci terraform (#59) * add terraform stage to GitHub Actions workflow * fix github.sha spelling * fix config tar permissions * add cd terraform to CI * Insert terraform init step * add hcloud SSH key in CI * add missing create --- .github/workflows/packer.yaml | 91 ++++++++++++++++++++++++++++--- docker-compose.yml | 4 +- terraform/terraform.tfvars.sample | 14 ++--- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/.github/workflows/packer.yaml b/.github/workflows/packer.yaml index 6d5b17f..24fdb3b 100644 --- a/.github/workflows/packer.yaml +++ b/.github/workflows/packer.yaml @@ -11,8 +11,6 @@ on: branches: - main workflow_dispatch: - schedule: - - cron: "0 1 * * 4" jobs: docker-build: @@ -24,12 +22,41 @@ jobs: run: | docker build -t ghcr.io/enowars/bambictf:latest . docker save --output /tmp/bambictf.tar ghcr.io/enowars/bambictf:latest - - name: Upload artifact + - name: Upload docker image artifact uses: actions/upload-artifact@v4 with: name: bambictf path: /tmp/bambictf.tar + configgen: + runs-on: ubuntu-latest + needs: docker-build + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: bambictf + path: /tmp + - name: Load Docker image + run: docker load --input /tmp/bambictf.tar + - name: start docker compose setup + env: + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + run: docker compose up -d + - name: prepare ansible config + run: docker compose exec -T bambictf sh -c 'cp ansible/config_bambi.yml.sample ansible/config_bambi.yml' + - name: generate config files + run: docker compose exec -T bambictf sh -c 'cd configgen; poetry install; poetry run configgen --teams 4 --routers 2 --dns ci-${{ github.sha }}.bambi.ovh' + - name: tar config files + run: sudo tar cf /tmp/config.tar config + - name: Upload config artifact + uses: actions/upload-artifact@v4 + with: + name: config + path: /tmp/config.tar + packer-build: strategy: # fail-fast needs to be disabled, otherwise packer processes may be ungracefully killed and leave stale builder VMs @@ -37,24 +64,74 @@ jobs: matrix: image: ["bambichecker", "bambielk", "bambiengine", "bambirouter", "bambivulnbox"] runs-on: ubuntu-latest - needs: docker-build + needs: configgen steps: - name: Checkout uses: actions/checkout@v4 - - name: Download artifact + - name: Download docker image artifact uses: actions/download-artifact@v4 with: name: bambictf path: /tmp - name: Load Docker image run: docker load --input /tmp/bambictf.tar + - name: Download config artifact + uses: actions/download-artifact@v4 + with: + name: config + path: /tmp + - name: untar config files + run: tar xf /tmp/config.tar - name: start docker compose setup env: HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} run: docker compose up -d - name: prepare ansible config run: docker compose exec -T bambictf sh -c 'cp ansible/config_bambi.yml.sample ansible/config_bambi.yml' - - name: generate config files - run: docker compose exec -T bambictf sh -c 'cd configgen; poetry install; poetry run configgen --teams 4 --routers 2 --dns bambi.ovh' - name: build packer image run: docker compose exec -T bambictf sh -c 'cd packer; packer build ${{ matrix.image }}.json' + + terraform-apply: + runs-on: ubuntu-latest + needs: packer-build + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download docker image artifact + uses: actions/download-artifact@v4 + with: + name: bambictf + path: /tmp + - name: Load Docker image + run: docker load --input /tmp/bambictf.tar + - name: Download config artifact + uses: actions/download-artifact@v4 + with: + name: config + path: /tmp + - name: untar config files + run: tar xf /tmp/config.tar + - name: copy terraform sample config + run: cp terraform/terraform.tfvars.sample terraform/terraform.tfvars + - name: prepare terraform config + run: sed -i s/prod/ci-${{ github.sha }}/ terraform/terraform.tfvars + - name: setup hcloud + run: | + curl -o hcloud-linux-amd64.tar.gz -L https://github.com/hetznercloud/cli/releases/download/v1.34.0/hcloud-linux-amd64.tar.gz + sudo tar xf hcloud-linux-amd64.tar.gz -C /usr/bin hcloud + sudo chmod +x /usr/bin/hcloud + - name: generate admin SSH key + run: ssh-keygen -t ed25519 -N "" -f adminkey + - name: add hcloud ssh key + env: + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + run: hcloud ssh-key create --name adminkey-${{ github.sha }} --label type=admin --public-key-from-file adminkey.pub + - name: start docker compose setup + env: + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + HETZNERDNS_TOKEN: ${{ secrets.HETZNERDNS_TOKEN }} + run: docker compose up -d + - name: perform terraform init + run: docker compose exec -T bambictf sh -c 'cd terraform ; terraform init' + - name: perform terraform apply + run: docker compose exec -T bambictf sh -c 'cd terraform ; terraform plan' diff --git a/docker-compose.yml b/docker-compose.yml index d610cca..643f376 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,8 @@ version: '2' services: bambictf: - #image: ghcr.io/enowars/bambictf:latest - build: . + image: ghcr.io/enowars/bambictf:latest + #build: . environment: - HCLOUD_TOKEN=${HCLOUD_TOKEN} # for packer - TF_VAR_HCLOUD_TOKEN=${HCLOUD_TOKEN} # for terraform diff --git a/terraform/terraform.tfvars.sample b/terraform/terraform.tfvars.sample index e42a6f7..92ffed6 100644 --- a/terraform/terraform.tfvars.sample +++ b/terraform/terraform.tfvars.sample @@ -1,15 +1,15 @@ hetznerdns_zone = "bambi.ovh" -hetznerdns_suffix = "prod" +subdomain = "prod" home_location = "fsn1" router_count = 2 checker_count = 1 engine_count = 1 elk_count = 1 -vulnbox_count = 0 +vulnbox_count = 2 -gateway_type = "cpx31" -checker_type = "cpx31" -engine_type = "cpx31" -elk_type = "cpx31" -vulnbox_type = "cpx21" +gateway_type = "cx21" +checker_type = "cx21" +engine_type = "cx21" +elk_type = "cx21" +vulnbox_type = "cx21" From 418b472662dc5e25687670ca03ec30934e98600d Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 6 Apr 2024 21:19:07 +0200 Subject: [PATCH 31/61] wg conf cleanup --- ansible/roles/bambi-arkime/files/arkime-capture.sh | 2 +- ansible/roles/bambi-arkime/files/config.ini | 2 +- configgen/configgen/gen_wireguard_internal.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/bambi-arkime/files/arkime-capture.sh index 33ba5f3..729226d 100755 --- a/ansible/roles/bambi-arkime/files/arkime-capture.sh +++ b/ansible/roles/bambi-arkime/files/arkime-capture.sh @@ -2,6 +2,6 @@ cd /opt/arkime while : do echo "Starting Arkime capture" - /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /pcaps -m --host $1 -n $1 + /opt/arkime/bin/capture -c /opt/arkime/etc/config.ini -R /opt/arkime/raw -m --host $1 -n $1 -d sleep 5 done diff --git a/ansible/roles/bambi-arkime/files/config.ini b/ansible/roles/bambi-arkime/files/config.ini index 1dd670e..24a25d4 100644 --- a/ansible/roles/bambi-arkime/files/config.ini +++ b/ansible/roles/bambi-arkime/files/config.ini @@ -11,7 +11,7 @@ httpRealm=Moloch # Semicolon ';' seperated list of interfaces to listen on for traffic interface=eth1 # The directory to save raw pcap files to -pcapDir=/pcaps +pcapDir=/opt/arkime/raw # The max raw pcap file size in gigabytes, with a max value of 36G. # The disk should have room for at least 10*maxFileSizeG diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index edbb164..17cc423 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -128,9 +128,9 @@ def gen_wireguard_internal( f"{DATA_DIR}/export/ansible/checkers/checker{checker_config.checker_id}.conf" ).write_text(create_config_file(checker_config)) - Path(f"{DATA_DIR}/export/ansible/engine/engine.conf").write_text( + Path(f"{DATA_DIR}/export/ansible/engine/internal.conf").write_text( create_config_file(engine_config) ) - Path(f"{DATA_DIR}/export/ansible/elk/elk.conf").write_text( + Path(f"{DATA_DIR}/export/ansible/elk/internal.conf").write_text( create_config_file(elk_config) ) From 611d48890132882bc71621abd0d8e110fc7ff68e Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 7 Apr 2024 00:32:25 +0200 Subject: [PATCH 32/61] always create elk and engine fips --- terraform/bambichecker.tf | 4 ++-- terraform/bambielk.tf | 7 +++---- terraform/bambiengine.tf | 9 ++++----- terraform/bambirouter.tf | 4 ++-- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index 77f7457..32a6408 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -37,8 +37,8 @@ resource "hcloud_server" "bambichecker" { "user_data_checker.tftpl", { id = "${count.index + 1}", router_ips = hcloud_floating_ip.bambirouter_ip, - elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", - engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + elk = hcloud_floating_ip.bambielk_ip.ip_address, + engine = hcloud_floating_ip.bambiengine_ip.ip_address, } ) } diff --git a/terraform/bambielk.tf b/terraform/bambielk.tf index 2dd8ea6..13a5ee7 100644 --- a/terraform/bambielk.tf +++ b/terraform/bambielk.tf @@ -22,7 +22,6 @@ data "hcloud_image" "bambielk" { } resource "hcloud_floating_ip" "bambielk_ip" { - count = var.elk_count type = "ipv4" name = "elk" home_location = var.home_location @@ -30,7 +29,7 @@ resource "hcloud_floating_ip" "bambielk_ip" { resource "hcloud_floating_ip_assignment" "bambielk_ipa" { count = var.elk_count - floating_ip_id = hcloud_floating_ip.bambielk_ip[0].id + floating_ip_id = hcloud_floating_ip.bambielk_ip.id server_id = hcloud_server.bambielk[0].id } @@ -54,8 +53,8 @@ resource "hcloud_server" "bambielk" { user_data = templatefile( "user_data_elk.tftpl", { router_ips = hcloud_floating_ip.bambirouter_ip, - elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", - engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + elk = hcloud_floating_ip.bambielk_ip.ip_address, + engine = hcloud_floating_ip.bambiengine_ip.ip_address, } ) } diff --git a/terraform/bambiengine.tf b/terraform/bambiengine.tf index 19baa2d..8aae16c 100644 --- a/terraform/bambiengine.tf +++ b/terraform/bambiengine.tf @@ -22,7 +22,6 @@ data "hcloud_image" "bambiengine" { } resource "hcloud_floating_ip" "bambiengine_ip" { - count = var.engine_count type = "ipv4" name = "engine" home_location = var.home_location @@ -30,7 +29,7 @@ resource "hcloud_floating_ip" "bambiengine_ip" { resource "hcloud_floating_ip_assignment" "bambiengine_ipa" { count = var.engine_count - floating_ip_id = hcloud_floating_ip.bambiengine_ip[0].id + floating_ip_id = hcloud_floating_ip.bambiengine_ip.id server_id = hcloud_server.bambiengine[0].id } @@ -38,7 +37,7 @@ resource "hetznerdns_record" "bambiengine_dns" { count = var.hetznerdns_zone != null ? var.engine_count : 0 zone_id = data.hetznerdns_zone.zone[0].id name = "engine${local.subdomain}" - value = hcloud_floating_ip.bambiengine_ip[0].ip_address + value = hcloud_floating_ip.bambiengine_ip.ip_address type = "A" ttl = 60 } @@ -54,8 +53,8 @@ resource "hcloud_server" "bambiengine" { user_data = templatefile( "user_data_engine.tftpl", { router_ips = hcloud_floating_ip.bambirouter_ip, - elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", - engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + elk = hcloud_floating_ip.bambielk_ip.ip_address, + engine = hcloud_floating_ip.bambiengine_ip.ip_address, } ) } diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf index 7448dab..1e96600 100644 --- a/terraform/bambirouter.tf +++ b/terraform/bambirouter.tf @@ -57,8 +57,8 @@ resource "hcloud_server" "bambirouter" { masters = join(",", [for i in range(var.router_count) : cidrhost(local.subnet, i+1)]), seeds = join(",", setsubtract([for i in range(var.router_count) : cidrhost(local.subnet, i+1)], [cidrhost(local.subnet, count.index+1)])) router_ips = hcloud_floating_ip.bambirouter_ip, - elk = var.elk_count > 0 ? hcloud_floating_ip.bambielk_ip[0].ip_address : "127.0.0.1", - engine = var.engine_count > 0 ? hcloud_floating_ip.bambiengine_ip[0].ip_address : "127.0.0.1", + elk = hcloud_floating_ip.bambielk_ip.ip_address, + engine = hcloud_floating_ip.bambiengine_ip.ip_address, } ) } From 2b99bbab97da87ba32929d7bf047e6fdbc168017 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Mon, 8 Apr 2024 00:40:01 +0200 Subject: [PATCH 33/61] =?UTF-8?q?arkime=20w=C3=BCrgarounds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + README.md | 1 + .../files/docker-compose.arkime.yml | 2 +- ansible/roles/enomoloch/tasks/main.yml | 4 +- .../files/enorouterdump.service | 2 +- .../router_trafficcapture/files/move_pcap.sh | 12 ++ .../router_trafficcapture/tasks/main.yml | 114 +++--------------- docker-compose.yml | 1 - terraform/terraform.tfvars.sample | 4 +- 9 files changed, 40 insertions(+), 101 deletions(-) create mode 100644 ansible/roles/router_trafficcapture/files/move_pcap.sh diff --git a/.gitignore b/.gitignore index d50bcdb..091c4b4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ id_ed25519 .secrets ctf*json *.pyc +/docker-compose.override.yml diff --git a/README.md b/README.md index b493fea..6d7bb05 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ export HCLOUD_TOKEN="..." - Obtain a private ssh ed25519 key that can clone your repositories (`cp ~/.ssh/id_ed25519 .`) - Run the container (`docker compose up -d`) - Invoke a bash in the container (`docker compose exec bambictf bash`) +- If you use Windows: Fix the private key permissions with `chmod 400 ./id_ed25519` - Build configs - `cd /bambictf/configgen` - `poetry install` (once) diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml index a9e93fd..38082bc 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml @@ -6,6 +6,6 @@ services: ports: - 8005:8005 volumes: - - "/pcaps:/opt/arkime/raw" + - "/pcaps_arkime:/opt/arkime/raw" - ".:/BambiArkime:ro" - "./config.ini:/opt/arkime/etc/config.ini:ro" diff --git a/ansible/roles/enomoloch/tasks/main.yml b/ansible/roles/enomoloch/tasks/main.yml index 0257e1a..f18d03b 100644 --- a/ansible/roles/enomoloch/tasks/main.yml +++ b/ansible/roles/enomoloch/tasks/main.yml @@ -19,8 +19,8 @@ - name: Disable tcpdump apparmor profile file: - path: /etc/apparmor.d/disable/usr.sbin.tcpdump - src: /etc/apparmor.d/usr.sbin.tcpdump + path: /etc/apparmor.d/disable/usr.bin.tcpdump + src: /etc/apparmor.d/usr.bin.tcpdump state: link force: yes follow: no diff --git a/ansible/roles/router_trafficcapture/files/enorouterdump.service b/ansible/roles/router_trafficcapture/files/enorouterdump.service index 3ec4f7e..a68de2c 100644 --- a/ansible/roles/router_trafficcapture/files/enorouterdump.service +++ b/ansible/roles/router_trafficcapture/files/enorouterdump.service @@ -8,7 +8,7 @@ RestartSec=1 TimeoutStartSec=300 WorkingDirectory=/pcaps -ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 -z touch src net 10.0.0.0/16 and dst net 10.0.0.0/16 +ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 -z ./move_pcap.sh src net 10.0.0.0/16 and dst net 10.0.0.0/16 [Install] WantedBy=multi-user.target diff --git a/ansible/roles/router_trafficcapture/files/move_pcap.sh b/ansible/roles/router_trafficcapture/files/move_pcap.sh new file mode 100644 index 0000000..7ff0da5 --- /dev/null +++ b/ansible/roles/router_trafficcapture/files/move_pcap.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +PCAP_HEADER_SIZE=24 +FILESIZE=$(stat -c%s "$1") + +if [ "$FILESIZE" -gt "$PCAP_HEADER_SIZE" ] +then + echo "moving $1 ($FILESIZE bytes)" + mv $1 ../pcaps_arkime + touch ../pcaps_arkime/$1 +fi diff --git a/ansible/roles/router_trafficcapture/tasks/main.yml b/ansible/roles/router_trafficcapture/tasks/main.yml index 1d3488d..b13eddc 100644 --- a/ansible/roles/router_trafficcapture/tasks/main.yml +++ b/ansible/roles/router_trafficcapture/tasks/main.yml @@ -1,18 +1,25 @@ --- -- name: install nfs-server - apt: - name: - - nfs-server - -- name: Enable enorouterdump - service: - name: nfs-server - enabled: yes - -- name: "Create /pcaps directory" +- name: Ensure /pcaps exists file: path: /pcaps state: directory + owner: tcpdump + group: tcpdump + +- name: Ensure /pcaps_arkime exists + file: + path: /pcaps_arkime + state: directory + owner: tcpdump + group: tcpdump + +- name: Copy move_pcap.sh + copy: + src: move_pcap.sh + dest: /pcaps/move_pcap.sh + owner: tcpdump + group: tcpdump + mode: '0700' - name: Install tcpdump apt: @@ -27,8 +34,8 @@ - name: Disable tcpdump apparmor profile file: - path: /etc/apparmor.d/disable/usr.sbin.tcpdump - src: /etc/apparmor.d/usr.sbin.tcpdump + path: /etc/apparmor.d/disable/usr.bin.tcpdump + src: /etc/apparmor.d/usr.bin.tcpdump state: link force: yes follow: no @@ -42,84 +49,3 @@ service: name: enorouterdump enabled: yes - -- name: "add pcaps to /etc/exports" - lineinfile: - path: /etc/exports - line: "/pcaps 192.168.2.0/255.255.255.0(ro,subtree_check)" - -- name: "run exportfs -a" - shell: exportfs -a - -- name: "specify mountd port to always be 33599" - lineinfile: - path: /etc/default/nfs-kernel-server - regexp: "^RPCMOUNTDOPTS" - line: 'RPCMOUNTDOPTS="--manage-gids --port 33599"' - -- name: allow NFS traffic (tcp port 111) - iptables: - chain: INPUT - in_interface: internal - protocol: tcp - source: "192.168.2.0/24" - destination_port: "111" - ctstate: NEW - syn: match - jump: ACCEPT - -- name: allow NFS traffic (tcp port 2049) - iptables: - chain: INPUT - in_interface: internal - protocol: tcp - source: "192.168.2.0/24" - destination_port: "2049" - ctstate: NEW - syn: match - jump: ACCEPT - -- name: allow NFS traffic (udp port 111) - iptables: - chain: INPUT - in_interface: internal - protocol: udp - source: "192.168.2.0/24" - destination_port: "111" - ctstate: NEW - jump: ACCEPT - -- name: allow NFS traffic (udp port 2049) - iptables: - chain: INPUT - in_interface: internal - protocol: udp - source: "192.168.2.0/24" - destination_port: "2049" - ctstate: NEW - jump: ACCEPT - -- name: allow NFS mountd traffic (udp port 33599) - iptables: - chain: INPUT - in_interface: internal - protocol: udp - source: "192.168.2.0/24" - destination_port: "33599" - ctstate: NEW - jump: ACCEPT - -- name: allow NFS mountd traffic (tcp port 33599) - iptables: - chain: INPUT - in_interface: internal - protocol: tcp - source: "192.168.2.0/24" - destination_port: "33599" - ctstate: NEW - syn: match - jump: ACCEPT - -- name: persist iptables config - include_role: - name: "iptables-persistent" diff --git a/docker-compose.yml b/docker-compose.yml index 643f376..e131085 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ version: '2' services: bambictf: image: ghcr.io/enowars/bambictf:latest - #build: . environment: - HCLOUD_TOKEN=${HCLOUD_TOKEN} # for packer - TF_VAR_HCLOUD_TOKEN=${HCLOUD_TOKEN} # for terraform diff --git a/terraform/terraform.tfvars.sample b/terraform/terraform.tfvars.sample index 92ffed6..e3340a3 100644 --- a/terraform/terraform.tfvars.sample +++ b/terraform/terraform.tfvars.sample @@ -11,5 +11,5 @@ vulnbox_count = 2 gateway_type = "cx21" checker_type = "cx21" engine_type = "cx21" -elk_type = "cx21" -vulnbox_type = "cx21" +elk_type = "cpx31" +vulnbox_type = "cpx31" From 70cb40efa1cbc27714dce13ffec5f3a62b39ab89 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Mon, 8 Apr 2024 21:27:03 +0000 Subject: [PATCH 34/61] wait for green ES before initializing arkime --- docker-compose.yml | 2 -- terraform/user_data_router.tftpl | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e131085..7d2be45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '2' - services: bambictf: image: ghcr.io/enowars/bambictf:latest diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 87fbd7c..175dc98 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -43,11 +43,21 @@ sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkime/docker-entryp cd /services/BambiArkime/ docker compose build -while ! curl -sq "http://10.13.0.${id}:9200/_cat/health"; do - echo "Waiting for elastic search to start... TODO: wait until green!"; - sleep 3; +echo "Waiting for green elastic search"; +while : ; do + if health="$(curl -fsSL "http://10.13.0.${id}:9200/_cat/health?h=status")"; then + health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" + if [ "$health" = 'green' ]; then + echo "ES is green!" + break + fi + echo "ES is still $health" + sleep 3; + fi + sleep 1; done +# Router 1 initializes the Arkime cluster if [ ${id} -eq 1 ] then docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://10.13.0.${id}:9200" init From dedf84ba175b26094ac9a32bac7d9281fa89ee1f Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 14 Apr 2024 14:01:17 +0000 Subject: [PATCH 35/61] vulnbox: pw login, openvpn --- ansible/bambirouter.yml | 1 - ansible/bambivulnbox.yml | 2 ++ ansible/roles/root-password-login/tasks/main.yml | 8 ++++++++ .../{teamvpn-configs => vulnbox-openvpn}/meta/main.yml | 0 .../{teamvpn-configs => vulnbox-openvpn}/tasks/main.yml | 7 +------ configgen/configgen/__init__.py | 5 +---- configgen/configgen/gen_openvpn.py | 5 ++--- configgen/configgen/gen_wireguard_internal.py | 4 +--- terraform/bambivulnbox.tf | 1 + terraform/user_data_vulnbox.tftpl | 6 ++++++ 10 files changed, 22 insertions(+), 17 deletions(-) create mode 100644 ansible/roles/root-password-login/tasks/main.yml rename ansible/roles/{teamvpn-configs => vulnbox-openvpn}/meta/main.yml (100%) rename ansible/roles/{teamvpn-configs => vulnbox-openvpn}/tasks/main.yml (53%) diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index c10b7aa..af34818 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -13,5 +13,4 @@ - metricbeat - bambi-wireguard-router - router_trafficcapture - - teamvpn-configs - bambi-arkime diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index 4bb8243..1f27452 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -13,6 +13,8 @@ - docker-block-external - enomoloch - bambixploit + - root-password-login + - vulnbox-openvpn - role: programs vars: program_list: diff --git a/ansible/roles/root-password-login/tasks/main.yml b/ansible/roles/root-password-login/tasks/main.yml new file mode 100644 index 0000000..ac5f449 --- /dev/null +++ b/ansible/roles/root-password-login/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Configure sshd to allow root login with password + lineinfile: + dest: /etc/ssh/sshd_config + regexp: '^PermitRootLogin' + line: "PermitRootLogin yes" + state: present + backup: yes diff --git a/ansible/roles/teamvpn-configs/meta/main.yml b/ansible/roles/vulnbox-openvpn/meta/main.yml similarity index 100% rename from ansible/roles/teamvpn-configs/meta/main.yml rename to ansible/roles/vulnbox-openvpn/meta/main.yml diff --git a/ansible/roles/teamvpn-configs/tasks/main.yml b/ansible/roles/vulnbox-openvpn/tasks/main.yml similarity index 53% rename from ansible/roles/teamvpn-configs/tasks/main.yml rename to ansible/roles/vulnbox-openvpn/tasks/main.yml index 1623a57..848538b 100644 --- a/ansible/roles/teamvpn-configs/tasks/main.yml +++ b/ansible/roles/vulnbox-openvpn/tasks/main.yml @@ -1,14 +1,9 @@ --- -- name: copy zipped team VPN configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/routers/openvpn/" - dest: /etc/openvpn/server - - name: allow openvpn input traffic iptables: chain: INPUT in_interface: eth0 - destination_port: "30001:32000" + destination_port: "30001" protocol: udp jump: ACCEPT diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index 9bbf66b..8897947 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -47,15 +47,12 @@ def prepare_directories(teams: int) -> None: Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/elk").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/engine").mkdir(parents=True, exist_ok=True) - Path(f"{DATA_DIR}/export/ansible/routers/openvpn").mkdir( - parents=True, exist_ok=True - ) + Path(f"{DATA_DIR}/export/ansible/routers").mkdir(parents=True, exist_ok=True) for team in range(1, teams + 1): Path(f"{DATA_DIR}/export/portal/team{team}").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/terraform/team{team}").mkdir( parents=True, exist_ok=True ) - Path(f"{DATA_DIR}/export/terraform/teams").mkdir(parents=True, exist_ok=True) def gen_userdata_portal(teams: int) -> None: diff --git a/configgen/configgen/gen_openvpn.py b/configgen/configgen/gen_openvpn.py index 90e5e63..9dee56f 100644 --- a/configgen/configgen/gen_openvpn.py +++ b/configgen/configgen/gen_openvpn.py @@ -26,7 +26,6 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: x, y = _get_team_octets(team) TEAM_SUBNET_PREFIX = f"10.{x}.{y}" server_config = f"""port {port} -local 0.0.0.0 proto udp dev team{team} dev-type tun @@ -70,7 +69,7 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: """ client_config = f"""proto udp dev tun -remote router{(team - 1) % routers + 1}.{dns} {port} +remote team{team}.{dns} {port} resolv-retry infinite nobind @@ -97,7 +96,7 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: {config.client_key}</key> """ # Save to disk - Path(f"{DATA_DIR}/export/ansible/routers/openvpn/team{team}.conf").write_text( + Path(f"{DATA_DIR}/export/terraform/team{team}/team.conf").write_text( server_config ) Path(f"{DATA_DIR}/export/portal/team{team}/client.ovpn").write_text( diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index 17cc423..f3e3ef2 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -21,9 +21,7 @@ logger = logging.getLogger(__file__) -def gen_wireguard_internal( - teams: int, checkers: int, routers: int -) -> None: +def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: """ Generates the internal wireguard config files as needed, reusing keys (if present) """ diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf index 407f6b8..e77446f 100644 --- a/terraform/bambivulnbox.tf +++ b/terraform/bambivulnbox.tf @@ -36,6 +36,7 @@ resource "hcloud_server" "bambivulnbox" { user_data = templatefile( "user_data_vulnbox.tftpl", { wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), + openvpnconf = file("../config/export/terraform/team${count.index + 1}/team.conf"), index = count.index, id = "${count.index + 1}", router_ips = hcloud_floating_ip.bambirouter_ip diff --git a/terraform/user_data_vulnbox.tftpl b/terraform/user_data_vulnbox.tftpl index 2fa1047..d53d5f7 100644 --- a/terraform/user_data_vulnbox.tftpl +++ b/terraform/user_data_vulnbox.tftpl @@ -18,3 +18,9 @@ for service in $(ls /services/); do fi docker compose up -d & done + +# OpenVPN +cat > /etc/openvpn/server/team.conf <<EOF +${openvpnconf} +EOF +systemctl enable --now openvpn-server@team From dce03e11c37f15667dc791b46aaf06abacff84d1 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sun, 14 Apr 2024 16:37:59 +0000 Subject: [PATCH 36/61] start every team's vpn on every router (for now) --- ansible/bambirouter.yml | 1 + ansible/bambivulnbox.yml | 1 - ansible/openvpn-server.yml | 9 --------- .../meta/main.yml | 0 .../tasks/main.yml | 7 ++++++- configgen/configgen/__init__.py | 4 +++- configgen/configgen/gen_openvpn.py | 4 ++-- terraform/bambivulnbox.tf | 1 - terraform/user_data_router.tftpl | 16 ++++++++++++++++ terraform/user_data_vulnbox.tftpl | 6 ------ 10 files changed, 28 insertions(+), 21 deletions(-) delete mode 100644 ansible/openvpn-server.yml rename ansible/roles/{vulnbox-openvpn => router-openvpn}/meta/main.yml (100%) rename ansible/roles/{vulnbox-openvpn => router-openvpn}/tasks/main.yml (52%) diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index af34818..5df75a3 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -14,3 +14,4 @@ - bambi-wireguard-router - router_trafficcapture - bambi-arkime + - router-openvpn diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index 1f27452..c6551bd 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -14,7 +14,6 @@ - enomoloch - bambixploit - root-password-login - - vulnbox-openvpn - role: programs vars: program_list: diff --git a/ansible/openvpn-server.yml b/ansible/openvpn-server.yml deleted file mode 100644 index 73de7f6..0000000 --- a/ansible/openvpn-server.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - roles: - - bambi-openvpn-team-server diff --git a/ansible/roles/vulnbox-openvpn/meta/main.yml b/ansible/roles/router-openvpn/meta/main.yml similarity index 100% rename from ansible/roles/vulnbox-openvpn/meta/main.yml rename to ansible/roles/router-openvpn/meta/main.yml diff --git a/ansible/roles/vulnbox-openvpn/tasks/main.yml b/ansible/roles/router-openvpn/tasks/main.yml similarity index 52% rename from ansible/roles/vulnbox-openvpn/tasks/main.yml rename to ansible/roles/router-openvpn/tasks/main.yml index 848538b..8a10e7a 100644 --- a/ansible/roles/vulnbox-openvpn/tasks/main.yml +++ b/ansible/roles/router-openvpn/tasks/main.yml @@ -1,9 +1,14 @@ --- +- name: copy team openvpn server configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/routers/openvpn/" + dest: /etc/openvpn/server + - name: allow openvpn input traffic iptables: chain: INPUT in_interface: eth0 - destination_port: "30001" + destination_port: "30001:32000" protocol: udp jump: ACCEPT diff --git a/configgen/configgen/__init__.py b/configgen/configgen/__init__.py index 8897947..97ee02d 100644 --- a/configgen/configgen/__init__.py +++ b/configgen/configgen/__init__.py @@ -47,7 +47,9 @@ def prepare_directories(teams: int) -> None: Path(f"{DATA_DIR}/export/ansible/checkers").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/elk").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/ansible/engine").mkdir(parents=True, exist_ok=True) - Path(f"{DATA_DIR}/export/ansible/routers").mkdir(parents=True, exist_ok=True) + Path(f"{DATA_DIR}/export/ansible/routers/openvpn").mkdir( + parents=True, exist_ok=True + ) for team in range(1, teams + 1): Path(f"{DATA_DIR}/export/portal/team{team}").mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/export/terraform/team{team}").mkdir( diff --git a/configgen/configgen/gen_openvpn.py b/configgen/configgen/gen_openvpn.py index 9dee56f..94591e7 100644 --- a/configgen/configgen/gen_openvpn.py +++ b/configgen/configgen/gen_openvpn.py @@ -69,7 +69,7 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: """ client_config = f"""proto udp dev tun -remote team{team}.{dns} {port} +remote router{(team - 1) % routers + 1}.{dns} {port} resolv-retry infinite nobind @@ -96,7 +96,7 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: {config.client_key}</key> """ # Save to disk - Path(f"{DATA_DIR}/export/terraform/team{team}/team.conf").write_text( + Path(f"{DATA_DIR}/export/ansible/routers/openvpn/team{team}.conf").write_text( server_config ) Path(f"{DATA_DIR}/export/portal/team{team}/client.ovpn").write_text( diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf index e77446f..407f6b8 100644 --- a/terraform/bambivulnbox.tf +++ b/terraform/bambivulnbox.tf @@ -36,7 +36,6 @@ resource "hcloud_server" "bambivulnbox" { user_data = templatefile( "user_data_vulnbox.tftpl", { wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), - openvpnconf = file("../config/export/terraform/team${count.index + 1}/team.conf"), index = count.index, id = "${count.index + 1}", router_ips = hcloud_floating_ip.bambirouter_ip diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 175dc98..83c07f4 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -1,6 +1,8 @@ #!/bin/bash set -e + +# FIP cat > /etc/netplan/60-floating-ip.yaml <<EOF network: version: 2 @@ -11,6 +13,8 @@ network: EOF netplan apply + +# Wireguard %{ for router_index, router in router_ips ~} %{ if router_index != index } rm "/etc/wireguard/router${router_index+1}_internal.conf" @@ -29,6 +33,7 @@ sed -i -e "s#\[\[ELK_ADDRESS\]\]#${elk}#g" "/etc/wireguard/internal.conf" systemctl enable --now "wg-quick@internal" systemctl enable --now "wg-quick@router" + # Arkime # Start elasticsearch cluster sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml @@ -71,3 +76,14 @@ while ! curl -s --head --show-error --fail "http://10.13.0.${id}:9200/bambi-arki done docker compose up -d + + +# OpenVPN +prefix="/etc/openvpn/server/" +for i in /etc/openvpn/server/*.conf; +do + filename="$${i##$prefix}" + echo "local ${router_ips[index].ip_address}" >> "/etc/openvpn/server/$filename" + configname="$${filename%.conf}" + systemctl enable --now "openvpn-server@$configname" +done diff --git a/terraform/user_data_vulnbox.tftpl b/terraform/user_data_vulnbox.tftpl index d53d5f7..2fa1047 100644 --- a/terraform/user_data_vulnbox.tftpl +++ b/terraform/user_data_vulnbox.tftpl @@ -18,9 +18,3 @@ for service in $(ls /services/); do fi docker compose up -d & done - -# OpenVPN -cat > /etc/openvpn/server/team.conf <<EOF -${openvpnconf} -EOF -systemctl enable --now openvpn-server@team From 5d363ab4a961c33846b61065b691a7302d1ae149 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Mon, 15 Apr 2024 18:44:51 +0000 Subject: [PATCH 37/61] =?UTF-8?q?w=C3=B6rk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bambi-wireguard-router/tasks/main.yml | 20 +++++++++++++++++++ .../roles/bambixploit/files/bambixploit.json | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml index 60f1715..d8f52b1 100644 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ b/ansible/roles/bambi-wireguard-router/tasks/main.yml @@ -116,6 +116,26 @@ destination_port: "1337" jump: ACCEPT + - name: allow enoctfportal traffic from vulnboxes + iptables: + chain: FORWARD + in_interface: router + out_interface: internal + destination: 192.168.1.0 + protocol: tcp + destination_port: "5001" + jump: ACCEPT + + - name: allow enoctfportal traffic from vpn connections + iptables: + chain: FORWARD + in_interface: team+ + out_interface: internal + destination: 192.168.1.0 + protocol: tcp + destination_port: "5001" + jump: ACCEPT + - name: allow traffic from the internal network to teams iptables: chain: FORWARD diff --git a/ansible/roles/bambixploit/files/bambixploit.json b/ansible/roles/bambixploit/files/bambixploit.json index e6da106..3fb6d16 100644 --- a/ansible/roles/bambixploit/files/bambixploit.json +++ b/ansible/roles/bambixploit/files/bambixploit.json @@ -1,6 +1,6 @@ { "flagRegex": "ENO[A-Za-z0-9+\/=]{48}", - "targetsUrl": "https://bambi7.enoflag.de/api/data/ips", + "targetsUrl": "http://10.0.13.37:5001/api/data/ips", "submissionAddress": "10.0.13.37", "submissionPort": 1337, "interval": 50 From 88ba7d2def7aef6dd1866fcf78c2f868ee54a3f1 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 19 Apr 2024 14:22:50 +0000 Subject: [PATCH 38/61] fix vulnbox IP, use network mode host on router, enable router p2p routing on internal interface --- .../bambi-arkime/files/docker-compose.arkime.yml | 1 + .../files/docker-compose.elasticsearch.yml | 1 + configgen/configgen/gen_openvpn.py | 4 ++-- configgen/configgen/gen_wireguard_game.py | 3 ++- configgen/configgen/gen_wireguard_internal.py | 16 ++++++++++++++++ configgen/configgen/util.py | 10 +++++----- terraform/bambirouter.tf | 2 +- terraform/user_data_router.tftpl | 14 +++++++------- 8 files changed, 35 insertions(+), 16 deletions(-) diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml index 38082bc..2d163e2 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml @@ -9,3 +9,4 @@ services: - "/pcaps_arkime:/opt/arkime/raw" - ".:/BambiArkime:ro" - "./config.ini:/opt/arkime/etc/config.ini:ro" + network_mode: "host" diff --git a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml index 460c82c..65369e8 100644 --- a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml +++ b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml @@ -14,3 +14,4 @@ services: ports: - "9200:9200" - "9300:9300" + network_mode: "host" diff --git a/configgen/configgen/gen_openvpn.py b/configgen/configgen/gen_openvpn.py index 94591e7..07b9579 100644 --- a/configgen/configgen/gen_openvpn.py +++ b/configgen/configgen/gen_openvpn.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from pathlib import Path -from configgen.util import DATA_DIR, _get_team_octets, run_subprocess +from configgen.util import DATA_DIR, get_team_octets, run_subprocess logger = logging.getLogger(__file__) @@ -23,7 +23,7 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: for team in range(1, teams + 1): config = get_team_config(team) port = 30000 + team - x, y = _get_team_octets(team) + x, y = get_team_octets(team) TEAM_SUBNET_PREFIX = f"10.{x}.{y}" server_config = f"""port {port} proto udp diff --git a/configgen/configgen/gen_wireguard_game.py b/configgen/configgen/gen_wireguard_game.py index 49be354..7aa41c6 100644 --- a/configgen/configgen/gen_wireguard_game.py +++ b/configgen/configgen/gen_wireguard_game.py @@ -14,6 +14,7 @@ WireguardTeamConfig, create_config_file, get_router_cidr_game, + get_team_octets, get_vulnbox_cidr, get_wg, ) @@ -48,7 +49,7 @@ def gen_wireguard_game(teams: int, dns: Optional[str], routers: int) -> None: # Generate team configs and add the peers to router configs for team in range(1, teams + 1): private_key, public_key = get_wg(Path(f"wg_game/team{team}.key")) - x, y = (team // 250), (team % 250) + x, y = get_team_octets(team) router_index = (team - 1) % routers router_config = router_configs[router_index] if dns is not None: diff --git a/configgen/configgen/gen_wireguard_internal.py b/configgen/configgen/gen_wireguard_internal.py index f3e3ef2..59ce2d6 100644 --- a/configgen/configgen/gen_wireguard_internal.py +++ b/configgen/configgen/gen_wireguard_internal.py @@ -115,6 +115,22 @@ def gen_wireguard_internal(teams: int, checkers: int, routers: int) -> None: router_config.peers.append(checker_peer) checker_configs.append(checker_config) + # Add internal router peers + for i in range(routers): + for j in range(routers): + if i == j: + continue + router_i = router_configs[i] + router_j = router_configs[j] + router_i.peers.append( + Peer( + public_key=router_j.public_key, + allowed_ips=[get_router_cidr_internal(j+1)], + endpoint=f"[[ROUTER_ADDRESS_{router_j.router_id}]]:{WG_LISTEN_PORT_INTERNAL}", + comment=f"router{router_j.router_id}", + ) + ) + # Save all to disk for router_config in router_configs: Path( diff --git a/configgen/configgen/util.py b/configgen/configgen/util.py index 0d16f79..4e5a184 100644 --- a/configgen/configgen/util.py +++ b/configgen/configgen/util.py @@ -121,17 +121,17 @@ def get_wg(path: Path) -> Tuple[str, str]: return priv, exec_wg_pub(priv) -def _get_team_octets(team_id: int) -> Tuple[int, int]: - return (team_id // 250), (team_id % 250) +def get_team_octets(team_id: int) -> Tuple[int, int]: + return ((team_id - 1) // 250) + 1, ((team_id - 1) % 250 + 1) def get_vulnbox_cidr(team_id: int) -> str: - x, y = _get_team_octets(team_id) + x, y = get_team_octets(team_id) return TEAM_IP_PREFIX_TEMPLATE % (x, y) + "1/32" def get_vulnbox_ip(team_id: int) -> str: - x, y = _get_team_octets(team_id) + x, y = get_team_octets(team_id) return TEAM_IP_PREFIX_TEMPLATE % (x, y) + "1" @@ -140,7 +140,7 @@ def get_router_index(routers: int, team_id: int) -> int: def get_team_cidr(team_id: int) -> str: - x, y = _get_team_octets(team_id) + x, y = get_team_octets(team_id) return f"10.{x}.{y}.0/24" diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf index 1e96600..f4e7d5b 100644 --- a/terraform/bambirouter.tf +++ b/terraform/bambirouter.tf @@ -11,7 +11,7 @@ variable "router_count" { } locals { - subnet = "10.13.0.0/24" + subnet = "192.168.0.0/24" } data "hcloud_image" "bambirouter" { diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 83c07f4..6cc9bd6 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -36,21 +36,21 @@ systemctl enable --now "wg-quick@router" # Arkime # Start elasticsearch cluster -sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml +sed -i -e "s#\[\[ROUTER\]\]#192.168.0.${id}#g" /services/BambiArkimeElasticsearch/docker-compose.yml sed -i -e "s#\[\[INITIAL_MASTER_NODES\]\]#${masters}#g" /services/BambiArkimeElasticsearch/docker-compose.yml sed -i -e "s#\[\[SEED_HOSTS\]\]#${seeds}#g" /services/BambiArkimeElasticsearch/docker-compose.yml cd /services/BambiArkimeElasticsearch/ docker compose up -d # Start Arkime cluster -sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkime/config.ini -sed -i -e "s#\[\[ROUTER\]\]#10.13.0.${id}#g" /services/BambiArkime/docker-entrypoint.sh +sed -i -e "s#\[\[ROUTER\]\]#192.168.0.${id}#g" /services/BambiArkime/config.ini +sed -i -e "s#\[\[ROUTER\]\]#192.168.0.${id}#g" /services/BambiArkime/docker-entrypoint.sh cd /services/BambiArkime/ docker compose build echo "Waiting for green elastic search"; while : ; do - if health="$(curl -fsSL "http://10.13.0.${id}:9200/_cat/health?h=status")"; then + if health="$(curl -fsSL "http://192.168.0.${id}:9200/_cat/health?h=status")"; then health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" if [ "$health" = 'green' ]; then echo "ES is green!" @@ -65,12 +65,12 @@ done # Router 1 initializes the Arkime cluster if [ ${id} -eq 1 ] then - docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://10.13.0.${id}:9200" init + docker compose run --entrypoint="/opt/arkime/db/db.pl" arkime "http://192.168.0.${id}:9200" init docker compose run --entrypoint="/opt/arkime/bin/arkime_add_user.sh" arkime admin "Admin" admin --admin - curl -X PUT "http://10.13.0.${id}:9200/bambi-arkime-init?pretty" + curl -X PUT "http://192.168.0.${id}:9200/bambi-arkime-init?pretty" fi -while ! curl -s --head --show-error --fail "http://10.13.0.${id}:9200/bambi-arkime-init/"; do +while ! curl -s --head --show-error --fail "http://192.168.0.${id}:9200/bambi-arkime-init/"; do echo "Waiting for marker index to exist..."; sleep 3; done From 9a46d7bd986bcb941cb3d94924d272746daf92cf Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 20 Apr 2024 14:17:36 +0000 Subject: [PATCH 39/61] I hate numbers --- ansible/roles/router_trafficcapture/files/enorouterdump.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/router_trafficcapture/files/enorouterdump.service b/ansible/roles/router_trafficcapture/files/enorouterdump.service index a68de2c..6542a31 100644 --- a/ansible/roles/router_trafficcapture/files/enorouterdump.service +++ b/ansible/roles/router_trafficcapture/files/enorouterdump.service @@ -8,7 +8,7 @@ RestartSec=1 TimeoutStartSec=300 WorkingDirectory=/pcaps -ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 -z ./move_pcap.sh src net 10.0.0.0/16 and dst net 10.0.0.0/16 +ExecStart=/usr/bin/tcpdump -i router -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 -z ./move_pcap.sh src net 10.1.0.0/16 and dst net 10.1.0.0/16 [Install] WantedBy=multi-user.target From 003da3effa1483ccb07d432d5994ca0963579ea3 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Sat, 20 Apr 2024 14:49:16 +0000 Subject: [PATCH 40/61] fix moloch cluster under host network mode --- ansible/roles/bambi-arkime/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/bambi-arkime/tasks/main.yml index c6a9a0b..8f08a26 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/bambi-arkime/tasks/main.yml @@ -10,6 +10,7 @@ # TODO fixme @ldruschk - shell: iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT - shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 9300 -j ACCEPT +- shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 8005 -j ACCEPT - name: persist iptables config include_role: From 6471bcd74b5b9dfe14ba4aeca5a77097a2910bd4 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 6 Jun 2024 08:42:40 +0000 Subject: [PATCH 41/61] cleanup documentation --- README.md | 50 +++++--------------------------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 6d7bb05..77de0b8 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,10 @@ This setup combines a lot of other services/repositories. - Moloch (Traffic Analysis): [EnoMoloch](https://github.com/enoflag/EnoMoloch) - ELK (Log Analysis): [EnoELK](https://github.com/enowars/EnoELK) -## Usage - -1. Create `./ansible/config_bambi.yml`: +## Usage (Docker) +- Have at least one ssh key with the label `type=admin` in your project **(HETZNER's WEBSITE)** +- Set `HCLOUD_TOKEN` and `HETZNERDNS_TOKEN` +- Create `./ansible/config_bambi.yml` ```yaml vulnerable_services: WASP: git@github.com:enowars/service-wasp.git @@ -31,45 +32,6 @@ github_ssh_keys: - ldruschk - MMunier ``` -2. Create `./terraform/terraform.tfvars`: -``` -hcloud_token = "..." -ovh_dyndns_password = "..." -``` -3. Initialize terraform: -``` -(cd terraform; terraform init) -``` -4. Generate wireguard configs for the internal network -```sh -(cd ./config/internal_router; ./gen_keys.sh $CHECKERS_COUNT) -``` -5. Generate wireguard configs for the game network -```sh -(cd ./config/wireguard_router; ./gen_keys.sh $TEAMS_COUNT) -``` -6. Generate passwords for the vulnboxes: -```sh -(cd ./config/passwords; ./gen_passwords.sh $TEAMS_COUNT) -``` -7. Create SSH keys for router -> moloch -```sh -(ssh-keygen -t ed25519 -f ./config/moloch_keys/moloch_key -C "tcpdump@router") -``` -8. Build images -```sh -export HCLOUD_TOKEN="..." -(cd packer; packer build bambichecker.json) -(cd packer; packer build bambiengine.json) -(cd packer; packer build bambirouter.json) -(cd packer; packer build bambivulnbox.json) -(cd packer; packer build bambielk.json) -``` - -## Docker -- Have at least one ssh key with the label `type=admin` in your project **(HETZNER's WEBSITE)** -- Set `HCLOUD_TOKEN` and `HETZNERDNS_TOKEN` -- Create `./ansible/config_bambi.yml` - Obtain a private ssh ed25519 key that can clone your repositories (`cp ~/.ssh/id_ed25519 .`) - Run the container (`docker compose up -d`) - Invoke a bash in the container (`docker compose exec bambictf bash`) @@ -85,9 +47,7 @@ export HCLOUD_TOKEN="..." - `packer build bambichecker.json` - ... - Note down vulnbox snapshot id, pass to EnoCTFPortal (`curl -H "Authorization: Bearer $HCLOUD_TOKEN" 'https://api.hetzner.cloud/v1/images?type=snapshot'`) -- Create `./terraform/terraform.tfvars` - - set `vpn_floating_ip_only = false` - - set `internal_floating_ip_only = false` +- Create `./terraform/terraform.tfvars` (see `./terraform/terraform.tfvars.sample` for reference) - `cd /bambictf/terraform` - `terraform init` - `terraform apply` From 359459c82885c4fed52dafcfa4116e4dc8c611e4 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Thu, 6 Jun 2024 17:57:41 +0000 Subject: [PATCH 42/61] woopsie --- terraform/terraform.tfvars.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/terraform.tfvars.sample b/terraform/terraform.tfvars.sample index e3340a3..91f3154 100644 --- a/terraform/terraform.tfvars.sample +++ b/terraform/terraform.tfvars.sample @@ -8,7 +8,7 @@ engine_count = 1 elk_count = 1 vulnbox_count = 2 -gateway_type = "cx21" +router_type = "cx21" checker_type = "cx21" engine_type = "cx21" elk_type = "cpx31" From 5eb54d65f70cc9d94ff5c882cc3fe0bf9f4e9d68 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Sun, 9 Jun 2024 12:33:24 +0200 Subject: [PATCH 43/61] make Moloch accessible via team VPN (untested) --- README.md | 6 ++++++ ansible/roles/bambi-wireguard-router/tasks/main.yml | 12 ++++++------ ansible/roles/enomoloch/files/enodump.service | 3 ++- ansible/roles/enomoloch/tasks/main.yml | 1 + 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 77de0b8..4b2b0ed 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,12 @@ This setup combines a lot of other services/repositories. - Moloch (Traffic Analysis): [EnoMoloch](https://github.com/enoflag/EnoMoloch) - ELK (Log Analysis): [EnoELK](https://github.com/enowars/EnoELK) +## Notable Limits +Due to implementation details, currently you have to be aware of the following limits: +- number of teams: 250 +- number of routers: 255 +- ... + ## Usage (Docker) - Have at least one ssh key with the label `type=admin` in your project **(HETZNER's WEBSITE)** - Set `HCLOUD_TOKEN` and `HETZNERDNS_TOKEN` diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml index d8f52b1..435429f 100644 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ b/ansible/roles/bambi-wireguard-router/tasks/main.yml @@ -32,12 +32,6 @@ ctstate: [RELATED, ESTABLISHED] jump: ACCEPT - - name: "Enable ipv4 forward" - sysctl: - name: net.ipv4.ip_forward - value: '1' - sysctl_set: yes - - name: "Capture iptables-save output" shell: "iptables-save" register: iptablessave @@ -71,6 +65,12 @@ out_interface: internal jump: ACCEPT + # this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow + - name: "prevent masquerading for intra-team traffic part" + shell: "iptables -t nat -A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" + when: '"-A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" not in iptablessave.stdout' + with_sequence: start=1 end=255 + - name: masquerade outgoing traffic on the router interface iptables: table: nat diff --git a/ansible/roles/enomoloch/files/enodump.service b/ansible/roles/enomoloch/files/enodump.service index 991f878..9da632b 100644 --- a/ansible/roles/enomoloch/files/enodump.service +++ b/ansible/roles/enomoloch/files/enodump.service @@ -8,7 +8,8 @@ Restart=always TimeoutStartSec=300 WorkingDirectory=/pcaps -ExecStart=/usr/bin/tcpdump -i game -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 +# all incoming traffic will appear to come from a 10.13.0.0/24 address +ExecStart=/usr/bin/tcpdump -i game -G 30 -w '%%Y_%%m_%%d-%%H_%%M_%%S.pcap' -s 0 ip net 10.13.0.0/24 [Install] WantedBy=multi-user.target diff --git a/ansible/roles/enomoloch/tasks/main.yml b/ansible/roles/enomoloch/tasks/main.yml index f18d03b..00ccb83 100644 --- a/ansible/roles/enomoloch/tasks/main.yml +++ b/ansible/roles/enomoloch/tasks/main.yml @@ -58,6 +58,7 @@ chain: DOCKER-USER in_interface: game protocol: tcp + source: "10.13.0.0/24" destination_port: "8005" jump: DROP From 31b3dd965da1630312c7e6b23fae31787e493864 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Sun, 9 Jun 2024 12:43:09 +0200 Subject: [PATCH 44/61] start only the necessary OpenVPN servers on each router (untested) --- configgen/configgen/gen_openvpn.py | 2 +- terraform/user_data_router.tftpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configgen/configgen/gen_openvpn.py b/configgen/configgen/gen_openvpn.py index 07b9579..8804054 100644 --- a/configgen/configgen/gen_openvpn.py +++ b/configgen/configgen/gen_openvpn.py @@ -96,7 +96,7 @@ def gen_openvpn(teams: int, routers: int, dns: str) -> None: {config.client_key}</key> """ # Save to disk - Path(f"{DATA_DIR}/export/ansible/routers/openvpn/team{team}.conf").write_text( + Path(f"{DATA_DIR}/export/ansible/routers/openvpn/router{(team - 1) % routers + 1}_team{team}.conf").write_text( server_config ) Path(f"{DATA_DIR}/export/portal/team{team}/client.ovpn").write_text( diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 6cc9bd6..1f96060 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -80,7 +80,7 @@ docker compose up -d # OpenVPN prefix="/etc/openvpn/server/" -for i in /etc/openvpn/server/*.conf; +for i in /etc/openvpn/server/router${router_index+1}_*.conf; do filename="$${i##$prefix}" echo "local ${router_ips[index].ip_address}" >> "/etc/openvpn/server/$filename" From bce077ebb2fbb6ba9ca945338ebce755345de413 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Sun, 9 Jun 2024 16:43:45 +0200 Subject: [PATCH 45/61] fix ansible when syntax --- .gitignore | 1 + ansible/roles/bambi-wireguard-router/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 091c4b4..e83b875 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ id_ed25519 ctf*json *.pyc /docker-compose.override.yml +.vscode/ diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml index 435429f..a2d589c 100644 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ b/ansible/roles/bambi-wireguard-router/tasks/main.yml @@ -68,7 +68,7 @@ # this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow - name: "prevent masquerading for intra-team traffic part" shell: "iptables -t nat -A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" - when: '"-A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" not in iptablessave.stdout' + when: '"-A POSTROUTING -s 10.1."+item"+.0/24 -d 10.1."+item+".0/24 -o router -j RETURN" not in iptablessave.stdout' with_sequence: start=1 end=255 - name: masquerade outgoing traffic on the router interface From 7567fb017fca1b7ba89a8ceb3766da2189104a0f Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Sun, 9 Jun 2024 17:11:03 +0200 Subject: [PATCH 46/61] fix typo --- ansible/roles/bambi-wireguard-router/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml index a2d589c..3d89a6e 100644 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ b/ansible/roles/bambi-wireguard-router/tasks/main.yml @@ -68,7 +68,7 @@ # this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow - name: "prevent masquerading for intra-team traffic part" shell: "iptables -t nat -A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" - when: '"-A POSTROUTING -s 10.1."+item"+.0/24 -d 10.1."+item+".0/24 -o router -j RETURN" not in iptablessave.stdout' + when: '"-A POSTROUTING -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j RETURN" not in iptablessave.stdout' with_sequence: start=1 end=255 - name: masquerade outgoing traffic on the router interface From e88bf4979154728559a29d44812aa8d98a600d23 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Sun, 9 Jun 2024 17:34:26 +0200 Subject: [PATCH 47/61] fix user_data_router.tftpl --- terraform/user_data_router.tftpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/user_data_router.tftpl b/terraform/user_data_router.tftpl index 1f96060..82e1d71 100644 --- a/terraform/user_data_router.tftpl +++ b/terraform/user_data_router.tftpl @@ -80,7 +80,7 @@ docker compose up -d # OpenVPN prefix="/etc/openvpn/server/" -for i in /etc/openvpn/server/router${router_index+1}_*.conf; +for i in /etc/openvpn/server/router${index+1}_*.conf; do filename="$${i##$prefix}" echo "local ${router_ips[index].ip_address}" >> "/etc/openvpn/server/$filename" From 1aefcee6ba37025745e8e64ae34a609cbcc03825 Mon Sep 17 00:00:00 2001 From: ldruschk <14059613+ldruschk@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:42:31 +0200 Subject: [PATCH 48/61] Ansible lint (#60) * reformat files for ansible-lint * more ansible-lint * run GH action when merging into config cleanup * add ansible-lint workflow * fix ansible-lint workflow * add github actions workflow to publish image on push to main * update Dockerfile to ubuntu:24.04 * fix issue with pipefail in sh * fix ansible typo * inject docker dependency into ansible * try fixing failing CI * update packer workflow --- .github/workflows/packer.yaml | 46 ++++- Dockerfile | 4 +- ansible/.ansible-lint | 2 + ansible/.ansible-lint-ignore | 4 + ansible/bambichecker.yml | 33 +-- ansible/bambielk.yml | 33 +-- ansible/bambiengine.yml | 33 +-- ansible/bambirouter.yml | 34 +-- ansible/bambivulnbox.yml | 45 ++-- .../files/docker-compose.arkime.yml | 12 -- .../files/docker-compose.elasticsearch.yml | 17 -- ansible/roles/bambi-arkime/meta/main.yml | 3 - ansible/roles/bambi-ssh-keys/tasks/main.yml | 7 - .../bambi-wireguard-router/meta/main.yml | 7 - .../bambi-wireguard-router/tasks/main.yml | 159 -------------- ansible/roles/bambixploit/tasks/main.yml | 28 +-- ansible/roles/base/meta/main.yml | 8 +- ansible/roles/base/tasks/main.yml | 8 +- .../roles/docker-block-external/meta/main.yml | 2 - .../docker-block-external/tasks/main.yml | 24 --- ansible/roles/docker/tasks/main.yml | 18 +- .../roles/docker_block_external/meta/main.yml | 2 + .../docker_block_external/tasks/main.yml | 24 +++ ansible/roles/dotnetsdk/tasks/main.yml | 10 +- ansible/roles/enoelk/meta/main.yml | 12 +- ansible/roles/enoelk/tasks/main.yml | 54 ++--- ansible/roles/enoengine/meta/main.yml | 12 +- ansible/roles/enoengine/tasks/main.yml | 117 +++++------ ansible/roles/enomoloch/meta/main.yml | 2 - ansible/roles/enomoloch/tasks/main.yml | 67 ------ ansible/roles/filebeat/tasks/main.yml | 50 ++--- ansible/roles/firewall/tasks/main.yml | 195 ++++++++++-------- .../roles/iptables-persistent/meta/main.yml | 6 - .../roles/iptables-persistent/tasks/main.yml | 6 - .../roles/iptables_persistent/meta/main.yml | 6 + .../roles/iptables_persistent/tasks/main.yml | 6 + ansible/roles/journalbeat/tasks/main.yml | 52 ++--- ansible/roles/metricbeat/tasks/main.yml | 6 +- ansible/roles/pip-packages/meta/main.yml | 5 - ansible/roles/pip-packages/tasks/main.yml | 5 - ansible/roles/pip_packages/meta/main.yml | 5 + ansible/roles/pip_packages/tasks/main.yml | 5 + ansible/roles/programs/tasks/main.yml | 14 +- .../tasks/main.yml | 2 +- ansible/roles/router-openvpn/meta/main.yml | 5 - .../files/Dockerfile | 0 .../files/arkime-capture.sh | 0 .../files/arkime-viewer.sh | 0 .../files/config.ini | 0 .../files/docker-compose.arkime.yml | 12 ++ .../files/docker-compose.elasticsearch.yml | 17 ++ .../files/docker-entrypoint.sh | 0 ansible/roles/router_arkime/meta/main.yml | 3 + .../tasks/main.yml | 31 +-- ansible/roles/router_iptables/meta/main.yml | 7 + ansible/roles/router_iptables/tasks/main.yml | 154 ++++++++++++++ ansible/roles/router_openvpn/meta/main.yml | 5 + .../tasks/main.yml | 9 +- .../router_trafficcapture/tasks/main.yml | 15 +- ansible/roles/ssh_keys/tasks/main.yml | 7 + ansible/roles/tmux/meta/main.yml | 8 +- ansible/roles/tmux/tasks/main.yml | 9 +- ansible/roles/vuln_checkers/meta/main.yml | 12 +- ansible/roles/vuln_checkers/tasks/main.yml | 60 +++--- ansible/roles/vuln_services/meta/main.yml | 12 +- ansible/roles/vuln_services/tasks/main.yml | 59 +++--- .../files/docker-compose.yml | 0 .../files/enodump.service | 0 ansible/roles/vulnbox_enomoloch/meta/main.yml | 2 + .../roles/vulnbox_enomoloch/tasks/main.yml | 71 +++++++ ansible/roles/wireguard/meta/main.yml | 8 +- ansible/roles/wireguard/tasks/main.yml | 10 +- ansible/roles/wireguard_configs/meta/main.yml | 2 +- .../roles/wireguard_configs/tasks/main.yml | 9 +- 74 files changed, 907 insertions(+), 810 deletions(-) create mode 100644 ansible/.ansible-lint create mode 100644 ansible/.ansible-lint-ignore delete mode 100644 ansible/roles/bambi-arkime/files/docker-compose.arkime.yml delete mode 100644 ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml delete mode 100644 ansible/roles/bambi-arkime/meta/main.yml delete mode 100644 ansible/roles/bambi-ssh-keys/tasks/main.yml delete mode 100644 ansible/roles/bambi-wireguard-router/meta/main.yml delete mode 100644 ansible/roles/bambi-wireguard-router/tasks/main.yml delete mode 100644 ansible/roles/docker-block-external/meta/main.yml delete mode 100644 ansible/roles/docker-block-external/tasks/main.yml create mode 100644 ansible/roles/docker_block_external/meta/main.yml create mode 100644 ansible/roles/docker_block_external/tasks/main.yml delete mode 100644 ansible/roles/enomoloch/meta/main.yml delete mode 100644 ansible/roles/enomoloch/tasks/main.yml delete mode 100644 ansible/roles/iptables-persistent/meta/main.yml delete mode 100644 ansible/roles/iptables-persistent/tasks/main.yml create mode 100644 ansible/roles/iptables_persistent/meta/main.yml create mode 100644 ansible/roles/iptables_persistent/tasks/main.yml delete mode 100644 ansible/roles/pip-packages/meta/main.yml delete mode 100644 ansible/roles/pip-packages/tasks/main.yml create mode 100644 ansible/roles/pip_packages/meta/main.yml create mode 100644 ansible/roles/pip_packages/tasks/main.yml rename ansible/roles/{root-password-login => root_password_login}/tasks/main.yml (91%) delete mode 100644 ansible/roles/router-openvpn/meta/main.yml rename ansible/roles/{bambi-arkime => router_arkime}/files/Dockerfile (100%) rename ansible/roles/{bambi-arkime => router_arkime}/files/arkime-capture.sh (100%) rename ansible/roles/{bambi-arkime => router_arkime}/files/arkime-viewer.sh (100%) rename ansible/roles/{bambi-arkime => router_arkime}/files/config.ini (100%) create mode 100644 ansible/roles/router_arkime/files/docker-compose.arkime.yml create mode 100644 ansible/roles/router_arkime/files/docker-compose.elasticsearch.yml rename ansible/roles/{bambi-arkime => router_arkime}/files/docker-entrypoint.sh (100%) create mode 100644 ansible/roles/router_arkime/meta/main.yml rename ansible/roles/{bambi-arkime => router_arkime}/tasks/main.yml (76%) create mode 100644 ansible/roles/router_iptables/meta/main.yml create mode 100644 ansible/roles/router_iptables/tasks/main.yml create mode 100644 ansible/roles/router_openvpn/meta/main.yml rename ansible/roles/{router-openvpn => router_openvpn}/tasks/main.yml (61%) create mode 100644 ansible/roles/ssh_keys/tasks/main.yml rename ansible/roles/{enomoloch => vulnbox_enomoloch}/files/docker-compose.yml (100%) rename ansible/roles/{enomoloch => vulnbox_enomoloch}/files/enodump.service (100%) create mode 100644 ansible/roles/vulnbox_enomoloch/meta/main.yml create mode 100644 ansible/roles/vulnbox_enomoloch/tasks/main.yml diff --git a/.github/workflows/packer.yaml b/.github/workflows/packer.yaml index 24fdb3b..fe3f31b 100644 --- a/.github/workflows/packer.yaml +++ b/.github/workflows/packer.yaml @@ -28,6 +28,24 @@ jobs: name: bambictf path: /tmp/bambictf.tar + ansible-lint: + runs-on: ubuntu-latest + needs: docker-build + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: bambictf + path: /tmp + - name: Load Docker image + run: docker load --input /tmp/bambictf.tar + - name: start docker compose setup + run: docker compose up -d + - name: run ansible-lint + run: docker compose exec -T bambictf sh -c 'cd ansible && ansible-lint' + configgen: runs-on: ubuntu-latest needs: docker-build @@ -62,7 +80,14 @@ jobs: # fail-fast needs to be disabled, otherwise packer processes may be ungracefully killed and leave stale builder VMs fail-fast: false matrix: - image: ["bambichecker", "bambielk", "bambiengine", "bambirouter", "bambivulnbox"] + image: + [ + "bambichecker", + "bambielk", + "bambiengine", + "bambirouter", + "bambivulnbox", + ] runs-on: ubuntu-latest needs: configgen steps: @@ -135,3 +160,22 @@ jobs: run: docker compose exec -T bambictf sh -c 'cd terraform ; terraform init' - name: perform terraform apply run: docker compose exec -T bambictf sh -c 'cd terraform ; terraform plan' + + publish-docker-image: + runs-on: ubuntu-latest + needs: terraform-apply + if: github.ref == 'refs/heads/main' + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download docker image artifact + uses: actions/download-artifact@v4 + with: + name: bambictf + path: /tmp + - name: Load Docker image + run: docker load --input /tmp/bambictf.tar + - name: Log into GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + - name: Push image to GitHub Container Registry + run: docker image push ghcr.io/enowars/bambictf:latest diff --git a/Dockerfile b/Dockerfile index 5ec4ee7..cd5ad5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:23.10 +FROM ubuntu:24.04 # Core deps RUN apt-get update @@ -7,7 +7,7 @@ RUN apt-get install -y --no-install-recommends openssh-client rsync git less tmu software-properties-common gpg-agent pipx # for ansible and packer install # Poetry and Ansible -RUN pipx install poetry && pipx install --include-deps ansible +RUN pipx install poetry && pipx install --include-deps ansible && pipx inject ansible ansible-lint --include-apps --include-deps ENV PATH="/root/.local/bin:${PATH}" # Packer and Terraform diff --git a/ansible/.ansible-lint b/ansible/.ansible-lint new file mode 100644 index 0000000..aee946c --- /dev/null +++ b/ansible/.ansible-lint @@ -0,0 +1,2 @@ +--- +profile: safety diff --git a/ansible/.ansible-lint-ignore b/ansible/.ansible-lint-ignore new file mode 100644 index 0000000..1589e67 --- /dev/null +++ b/ansible/.ansible-lint-ignore @@ -0,0 +1,4 @@ +roles/router_iptables/tasks/main.yml yaml[line-length] +roles/router_iptables/tasks/main.yml command-instead-of-shell +roles/router_arkime/tasks/main.yml name[missing] +roles/docker/tasks/main.yml command-instead-of-module diff --git a/ansible/bambichecker.yml b/ansible/bambichecker.yml index fc3e87c..49b84b4 100644 --- a/ansible/bambichecker.yml +++ b/ansible/bambichecker.yml @@ -1,18 +1,19 @@ --- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - static.yml +- name: Bambi Checker + hosts: all + become: true + become_method: ansible.builtin.sudo + vars_files: + - config_bambi.yml + - static.yml - roles: - - base - - role: wireguard_configs - vars: - config_dir: checkers - - filebeat - - journalbeat - - metricbeat - - vuln_checkers - - docker-block-external + roles: + - base + - role: wireguard_configs + vars: + wireguard_configs_dir: checkers + - filebeat + - journalbeat + - metricbeat + - vuln_checkers + - docker_block_external diff --git a/ansible/bambielk.yml b/ansible/bambielk.yml index 90d6ad7..91bd37d 100644 --- a/ansible/bambielk.yml +++ b/ansible/bambielk.yml @@ -1,18 +1,19 @@ --- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - static.yml +- name: Bambi ELK + hosts: all + become: true + become_method: ansible.builtin.sudo + vars_files: + - config_bambi.yml + - static.yml - roles: - - base - - enoelk - - docker-block-external - - role: wireguard_configs - vars: - config_dir: elk - - filebeat - - journalbeat - - metricbeat + roles: + - base + - enoelk + - docker_block_external + - role: wireguard_configs + vars: + wireguard_configs_dir: elk + - filebeat + - journalbeat + - metricbeat diff --git a/ansible/bambiengine.yml b/ansible/bambiengine.yml index bc6c54b..a9004b9 100644 --- a/ansible/bambiengine.yml +++ b/ansible/bambiengine.yml @@ -1,18 +1,19 @@ --- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - static.yml +- name: Bambi Engine + hosts: all + become: true + become_method: ansible.builtin.sudo + vars_files: + - config_bambi.yml + - static.yml - roles: - - base - - enoengine - - role: wireguard_configs - vars: - config_dir: engine - - filebeat - - journalbeat - - metricbeat - - docker-block-external + roles: + - base + - enoengine + - role: wireguard_configs + vars: + wireguard_configs_dir: engine + - filebeat + - journalbeat + - metricbeat + - docker_block_external diff --git a/ansible/bambirouter.yml b/ansible/bambirouter.yml index 5df75a3..d6676d9 100644 --- a/ansible/bambirouter.yml +++ b/ansible/bambirouter.yml @@ -1,17 +1,21 @@ --- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - static.yml +- name: Bambi Router + hosts: all + become: true + become_method: ansible.builtin.sudo + vars_files: + - config_bambi.yml + - static.yml - roles: - - base - - filebeat - - journalbeat - - metricbeat - - bambi-wireguard-router - - router_trafficcapture - - bambi-arkime - - router-openvpn + roles: + - base + - filebeat + - journalbeat + - metricbeat + - role: wireguard_configs + vars: + wireguard_configs_dir: routers + - router_iptables + - router_trafficcapture + - router_arkime + - router_openvpn diff --git a/ansible/bambivulnbox.yml b/ansible/bambivulnbox.yml index c6551bd..9e5f27a 100644 --- a/ansible/bambivulnbox.yml +++ b/ansible/bambivulnbox.yml @@ -1,24 +1,25 @@ --- - - hosts: all - become: yes - become_method: sudo - vars_files: - - config_bambi.yml - - static.yml +- name: Bambi Vulnbox + hosts: all + become: true + become_method: ansible.builtin.sudo + vars_files: + - config_bambi.yml + - static.yml - roles: - - base - - docker - - vuln_services - - docker-block-external - - enomoloch - - bambixploit - - root-password-login - - role: programs - vars: - program_list: - - unzip - - role: pip-packages - vars: - pip_list: - - pwntools + roles: + - base + - docker + - vuln_services + - docker_block_external + - vulnbox_enomoloch + - bambixploit + - root_password_login + - role: programs + vars: + programs_list: + - unzip + - role: pip_packages + vars: + pip_packages_list: + - pwntools diff --git a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml b/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml deleted file mode 100644 index 2d163e2..0000000 --- a/ansible/roles/bambi-arkime/files/docker-compose.arkime.yml +++ /dev/null @@ -1,12 +0,0 @@ -services: - arkime: - #image: ghcr.io/enoflag/enoarkime:nightly - build: . - restart: unless-stopped - ports: - - 8005:8005 - volumes: - - "/pcaps_arkime:/opt/arkime/raw" - - ".:/BambiArkime:ro" - - "./config.ini:/opt/arkime/etc/config.ini:ro" - network_mode: "host" diff --git a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml b/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml deleted file mode 100644 index 65369e8..0000000 --- a/ansible/roles/bambi-arkime/files/docker-compose.elasticsearch.yml +++ /dev/null @@ -1,17 +0,0 @@ -services: - elasticsearch: - restart: unless-stopped - image: elasticsearch:8.11.1 - environment: - - xpack.security.enabled=false - - node.name=[[ROUTER]] - - cluster.name=bambiarkime - - cluster.initial_master_nodes=[[INITIAL_MASTER_NODES]] - - discovery.seed_hosts=[[SEED_HOSTS]] - - network.publish_host=[[ROUTER]] - - http.publish_host=[[ROUTER]] - - transport.publish_host=[[ROUTER]] - ports: - - "9200:9200" - - "9300:9300" - network_mode: "host" diff --git a/ansible/roles/bambi-arkime/meta/main.yml b/ansible/roles/bambi-arkime/meta/main.yml deleted file mode 100644 index 1e3630d..0000000 --- a/ansible/roles/bambi-arkime/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: -- role: "docker" -- role: "bambi-ssh-keys" diff --git a/ansible/roles/bambi-ssh-keys/tasks/main.yml b/ansible/roles/bambi-ssh-keys/tasks/main.yml deleted file mode 100644 index 7994f77..0000000 --- a/ansible/roles/bambi-ssh-keys/tasks/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - - name: Pull authkeys from github - authorized_key: - user: root - state: present - key: "https://github.com/{{item}}.keys" - with_items: "{{ github_ssh_keys }}" diff --git a/ansible/roles/bambi-wireguard-router/meta/main.yml b/ansible/roles/bambi-wireguard-router/meta/main.yml deleted file mode 100644 index 28fe50f..0000000 --- a/ansible/roles/bambi-wireguard-router/meta/main.yml +++ /dev/null @@ -1,7 +0,0 @@ -dependencies: -- role: "wireguard" -- role: "firewall" -- role: "programs" - vars: - program_list: - - iptables-persistent diff --git a/ansible/roles/bambi-wireguard-router/tasks/main.yml b/ansible/roles/bambi-wireguard-router/tasks/main.yml deleted file mode 100644 index 3d89a6e..0000000 --- a/ansible/roles/bambi-wireguard-router/tasks/main.yml +++ /dev/null @@ -1,159 +0,0 @@ ---- - - name: Copy wg configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/routers/" - dest: /etc/wireguard/ - - - name: allow wireguard input traffic (game) - iptables: - chain: INPUT - in_interface: eth0 - destination_port: "51820" - protocol: udp - jump: ACCEPT - - - name: allow wireguard input traffic (internal) - iptables: - chain: INPUT - in_interface: eth0 - destination_port: "51821" - protocol: udp - jump: ACCEPT - - - name: "Enable ipv4 forward" - sysctl: - name: net.ipv4.ip_forward - value: '1' - sysctl_set: yes - - - name: allow related/established forward traffic - iptables: - chain: FORWARD - ctstate: [RELATED, ESTABLISHED] - jump: ACCEPT - - - name: "Capture iptables-save output" - shell: "iptables-save" - register: iptablessave - - - name: "Insert limit bw internal" - shell: "iptables -I FORWARD -i router -o internal -m hashlimit --hashlimit-above 10mb/s --hashlimit-burst 20mb --hashlimit-mode srcip --hashlimit-name bwi --hashlimit-srcmask 24 -j DROP" - when: '"-A FORWARD -i router -o internal -m hashlimit --hashlimit-above 10mb/s --hashlimit-burst 20mb --hashlimit-mode srcip --hashlimit-name bwi --hashlimit-srcmask 24 -j DROP" not in iptablessave.stdout' - - - name: "Insert limit bw other" - shell: "iptables -I FORWARD -i router ! -o internal -m hashlimit --hashlimit-above 5mb/s --hashlimit-burst 10mb --hashlimit-mode srcip --hashlimit-name bw --hashlimit-srcmask 24 -j DROP" - when: '"-A FORWARD -i router ! -o internal -m hashlimit --hashlimit-above 5mb/s --hashlimit-burst 10mb --hashlimit-mode srcip --hashlimit-name bw --hashlimit-srcmask 24 -j DROP" not in iptablessave.stdout' - - - name: "Insert limit conn internal" - shell: "iptables -I FORWARD -i router -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 2/sec --hashlimit-burst 10 --hashlimit-mode srcip,dstip --hashlimit-name subnewconns -j DROP" - when: '"-A FORWARD -i router -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 2/sec --hashlimit-burst 10 --hashlimit-mode srcip,dstip --hashlimit-name subnewconns -j DROP" not in iptablessave.stdout' - - - name: "Insert limit conn other" - shell: "iptables -I FORWARD -i router ! -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 50/sec --hashlimit-burst 100 --hashlimit-mode srcip,dstip --hashlimit-name newconns -j DROP" - when: '"-A FORWARD -i router ! -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 50/sec --hashlimit-burst 100 --hashlimit-mode srcip,dstip --hashlimit-name newconns -j DROP" not in iptablessave.stdout' - - - name: allow related/established forward traffic - iptables: - chain: FORWARD - ctstate: [RELATED, ESTABLISHED] - jump: ACCEPT - - - name: allow all internal traffic - iptables: - chain: FORWARD - in_interface: internal - out_interface: internal - jump: ACCEPT - - # this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow - - name: "prevent masquerading for intra-team traffic part" - shell: "iptables -t nat -A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" - when: '"-A POSTROUTING -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j RETURN" not in iptablessave.stdout' - with_sequence: start=1 end=255 - - - name: masquerade outgoing traffic on the router interface - iptables: - table: nat - chain: POSTROUTING - out_interface: router - jump: MASQUERADE - - - name: DNAT the flag submission IP to the engine - iptables: - table: nat - chain: PREROUTING - destination: 10.0.13.37 - jump: DNAT - to_destination: 192.168.1.0 - - - name: allow pinging the flag submission endpoint - iptables: - chain: FORWARD - in_interface: router - out_interface: internal - destination: 192.168.1.0 - protocol: icmp - icmp_type: "8" - jump: ACCEPT - - - name: allow flag submission traffic from vulnboxes - iptables: - chain: FORWARD - in_interface: router - out_interface: internal - destination: 192.168.1.0 - protocol: tcp - destination_port: "1337" - jump: ACCEPT - - - name: allow flag submission traffic from vpn connections - iptables: - chain: FORWARD - in_interface: team+ - out_interface: internal - destination: 192.168.1.0 - protocol: tcp - destination_port: "1337" - jump: ACCEPT - - - name: allow enoctfportal traffic from vulnboxes - iptables: - chain: FORWARD - in_interface: router - out_interface: internal - destination: 192.168.1.0 - protocol: tcp - destination_port: "5001" - jump: ACCEPT - - - name: allow enoctfportal traffic from vpn connections - iptables: - chain: FORWARD - in_interface: team+ - out_interface: internal - destination: 192.168.1.0 - protocol: tcp - destination_port: "5001" - jump: ACCEPT - - - name: allow traffic from the internal network to teams - iptables: - chain: FORWARD - in_interface: internal - out_interface: router - jump: ACCEPT - - # this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow - - name: "allow intra-team traffic part 1" - shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j ACCEPT" - when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' - with_sequence: start=1 end=255 - - - name: "allow intra-team traffic part 2" - shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o team+ -j ACCEPT" - when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' - with_sequence: start=1 end=255 - - - name: persist iptables config - include_role: - name: "iptables-persistent" diff --git a/ansible/roles/bambixploit/tasks/main.yml b/ansible/roles/bambixploit/tasks/main.yml index 9b3bf41..7cfc978 100644 --- a/ansible/roles/bambixploit/tasks/main.yml +++ b/ansible/roles/bambixploit/tasks/main.yml @@ -1,16 +1,18 @@ --- - - name: Install bambixploit - get_url: - url: "https://github.com/enowars/bambiXploit-dotnet/releases/download/v0.0.10/bambixploit-linux-x64" - dest: /usr/local/bin/bambixploit - mode: 0755 +- name: Install bambixploit + get_url: + url: "https://github.com/enowars/bambiXploit-dotnet/releases/download/v0.0.10/bambixploit-linux-x64" + dest: /usr/local/bin/bambixploit + mode: "0755" - - name: Create /etc/bambixploit - ansible.builtin.file: - path: /etc/bambixploit - state: directory +- name: Create /etc/bambixploit + ansible.builtin.file: + path: /etc/bambixploit + state: directory + mode: "0755" - - name: Deploy bambixploit.json - copy: - src: bambixploit.json - dest: /etc/bambixploit/bambixploit.json +- name: Deploy bambixploit.json + copy: + src: bambixploit.json + dest: /etc/bambixploit/bambixploit.json + mode: "0644" diff --git a/ansible/roles/base/meta/main.yml b/ansible/roles/base/meta/main.yml index 2710153..06d70e4 100644 --- a/ansible/roles/base/meta/main.yml +++ b/ansible/roles/base/meta/main.yml @@ -1,5 +1,5 @@ dependencies: -- role: "bambi-ssh-keys" -- role: "firewall" -- role: "tmux" -- role: "wireguard" + - role: "ssh_keys" + - role: "firewall" + - role: "tmux" + - role: "wireguard" diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml index 4384fea..501aab0 100644 --- a/ansible/roles/base/tasks/main.yml +++ b/ansible/roles/base/tasks/main.yml @@ -1,5 +1,5 @@ --- - - name: Update and upgrade apt packages - apt: - upgrade: "yes" - update_cache: yes +- name: Update and upgrade apt packages + apt: + upgrade: true + update_cache: true diff --git a/ansible/roles/docker-block-external/meta/main.yml b/ansible/roles/docker-block-external/meta/main.yml deleted file mode 100644 index fc4a1e2..0000000 --- a/ansible/roles/docker-block-external/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: -- role: "docker" diff --git a/ansible/roles/docker-block-external/tasks/main.yml b/ansible/roles/docker-block-external/tasks/main.yml deleted file mode 100644 index f5ed6eb..0000000 --- a/ansible/roles/docker-block-external/tasks/main.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - # note that this is Hetzner Cloud specific and does not work if the external interface is not eth0 - - name: "Allow related/established traffic from the external interface to allow the containers to access the internet" - iptables: - chain: DOCKER-USER - ctstate: [RELATED, ESTABLISHED] - in_interface: eth0 - jump: ACCEPT - - - name: "Prevent access to docker services from the external interface" - iptables: - chain: DOCKER-USER - in_interface: eth0 - jump: DROP - - - name: "Delete RETURN rule in DOCKER-USER" - iptables: - chain: DOCKER-USER - jump: RETURN - state: absent - - - name: persist iptables config - include_role: - name: "iptables-persistent" diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index e29fb8f..612de20 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -1,7 +1,10 @@ --- - # https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository - - name: Install docker - shell: | +# https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository +- name: Install docker + shell: + executable: /usr/bin/bash + cmd: | + set -e -o pipefail apt-get update -y apt-get install -y ca-certificates curl gnupg install -m 0755 -d /etc/apt/keyrings @@ -17,7 +20,8 @@ apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - name: Copy daemon.json - copy: - src: daemon.json - dest: /etc/docker/daemon.json +- name: Copy daemon.json + copy: + src: daemon.json + dest: /etc/docker/daemon.json + mode: "0644" diff --git a/ansible/roles/docker_block_external/meta/main.yml b/ansible/roles/docker_block_external/meta/main.yml new file mode 100644 index 0000000..0e0662f --- /dev/null +++ b/ansible/roles/docker_block_external/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - role: "docker" diff --git a/ansible/roles/docker_block_external/tasks/main.yml b/ansible/roles/docker_block_external/tasks/main.yml new file mode 100644 index 0000000..bb39e56 --- /dev/null +++ b/ansible/roles/docker_block_external/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# note that this is Hetzner Cloud specific and does not work if the external interface is not eth0 +- name: "Allow related/established traffic from the external interface to allow the containers to access the internet" + iptables: + chain: DOCKER-USER + ctstate: [RELATED, ESTABLISHED] + in_interface: eth0 + jump: ACCEPT + +- name: "Prevent access to docker services from the external interface" + iptables: + chain: DOCKER-USER + in_interface: eth0 + jump: DROP + +- name: "Delete RETURN rule in DOCKER-USER" + iptables: + chain: DOCKER-USER + jump: RETURN + state: absent + +- name: Persist iptables config + include_role: + name: "iptables_persistent" diff --git a/ansible/roles/dotnetsdk/tasks/main.yml b/ansible/roles/dotnetsdk/tasks/main.yml index 9360467..25bceb5 100644 --- a/ansible/roles/dotnetsdk/tasks/main.yml +++ b/ansible/roles/dotnetsdk/tasks/main.yml @@ -1,6 +1,6 @@ --- - - name: Install dotnet sdk - apt: - name: "dotnet-sdk-6.0" - state: present - cache_valid_time: 250 +- name: Install dotnet sdk + apt: + name: "dotnet-sdk-6.0" + state: present + cache_valid_time: 250 diff --git a/ansible/roles/enoelk/meta/main.yml b/ansible/roles/enoelk/meta/main.yml index 18976ce..be249a1 100644 --- a/ansible/roles/enoelk/meta/main.yml +++ b/ansible/roles/enoelk/meta/main.yml @@ -1,6 +1,6 @@ -dependencies: -- role: "docker" -- role: "programs" - vars: - program_list: - - git +dependencies: + - role: "docker" + - role: "programs" + vars: + programs_list: + - git diff --git a/ansible/roles/enoelk/tasks/main.yml b/ansible/roles/enoelk/tasks/main.yml index 1597874..010be1a 100644 --- a/ansible/roles/enoelk/tasks/main.yml +++ b/ansible/roles/enoelk/tasks/main.yml @@ -1,33 +1,33 @@ --- - - name: Increase max_map_count - sysctl: - name: vm.max_map_count - value: "262144" - sysctl_set: yes +- name: Increase max_map_count + sysctl: + name: vm.max_map_count + value: "262144" + sysctl_set: true - - name: Install git dependency - apt: - name: ["git"] - state: present +- name: Install git dependency + apt: + name: ["git"] + state: present - - name: Clone EnoELK - git: - repo: "https://github.com/enowars/EnoELK.git" - dest: /services/EnoELK +- name: Clone EnoELK + git: # noqa: latest + repo: "https://github.com/enowars/EnoELK.git" + dest: /services/EnoELK - - name: Pull ELK images - shell: docker compose pull - args: - chdir: /services/EnoELK +- name: Pull ELK images + shell: docker compose pull # noqa: command-instead-of-shell + args: + chdir: /services/EnoELK - - name: allow wireguard input traffic (internal) - iptables: - chain: INPUT - in_interface: eth0 - destination_port: "51821" - protocol: udp - jump: ACCEPT +- name: Allow wireguard input traffic (internal) + iptables: + chain: INPUT + in_interface: eth0 + destination_port: "51821" + protocol: udp + jump: ACCEPT - - name: persist iptables config - include_role: - name: "iptables-persistent" +- name: Persist iptables config + include_role: + name: "iptables_persistent" diff --git a/ansible/roles/enoengine/meta/main.yml b/ansible/roles/enoengine/meta/main.yml index 8d209af..8e85ac7 100644 --- a/ansible/roles/enoengine/meta/main.yml +++ b/ansible/roles/enoengine/meta/main.yml @@ -1,7 +1,7 @@ dependencies: -- role: "dotnetsdk" -- role: "docker" -- role: "programs" - vars: - program_list: - - git + - role: "dotnetsdk" + - role: "docker" + - role: "programs" + vars: + programs_list: + - git diff --git a/ansible/roles/enoengine/tasks/main.yml b/ansible/roles/enoengine/tasks/main.yml index aa525d1..3abaf0a 100644 --- a/ansible/roles/enoengine/tasks/main.yml +++ b/ansible/roles/enoengine/tasks/main.yml @@ -1,70 +1,71 @@ --- - - name: Install docker dependencies - apt: - name: ["git"] - state: present +- name: Install docker dependencies + apt: + name: ["git"] + state: present - - name: Clone EnoEngine - git: - repo: "https://github.com/enowars/EnoEngine.git" - dest: /services/EnoEngine +- name: Clone EnoEngine + git: # noqa: latest + repo: "https://github.com/enowars/EnoEngine.git" + dest: /services/EnoEngine - - name: Build EnoEngine - shell: "dotnet build" - args: - chdir: /services/EnoEngine +- name: Build EnoEngine + shell: "dotnet build" # noqa: command-instead-of-shell + args: + chdir: /services/EnoEngine - - name: Start EnoEngine Postgres - shell: "docker compose up -d" - args: - chdir: /services/EnoEngine +- name: Start EnoEngine Postgres + shell: "docker compose up -d" # noqa: command-instead-of-shell + args: + chdir: /services/EnoEngine - - name: Clone EnoCTFPortal (for adhoc testing and solutions) - git: - repo: "https://github.com/enowars/EnoCTFPortal.git" - dest: /services/EnoCTFPortal +- name: Clone EnoCTFPortal (for adhoc testing and solutions) + git: # noqa: latest + repo: "https://github.com/enowars/EnoCTFPortal.git" + dest: /services/EnoCTFPortal - - name: Copy EnoCTFPortal docker-compose.yml - copy: - src: docker-compose.yml - dest: /services/EnoCTFPortal/ - changed_when: false +- name: Copy EnoCTFPortal docker-compose.yml + copy: + src: docker-compose.yml + dest: /services/EnoCTFPortal/ + mode: "0644" + changed_when: false - - name: Start EnoCTFPortal - shell: "docker compose up -d" - args: - chdir: /services/EnoCTFPortal +- name: Start EnoCTFPortal + shell: "docker compose up -d" # noqa: command-instead-of-shell + args: + chdir: /services/EnoCTFPortal - - name: Allow flag submission traffic - iptables: - chain: INPUT - in_interface: internal - protocol: tcp - destination_port: "1337" - jump: ACCEPT +- name: Allow flag submission traffic + iptables: + chain: INPUT + in_interface: internal + protocol: tcp + destination_port: "1337" + jump: ACCEPT - - name: Allow EnoCTFPortal traffic - iptables: - chain: INPUT - in_interface: internal - protocol: tcp - destination_port: "5001" - jump: ACCEPT +- name: Allow EnoCTFPortal traffic + iptables: + chain: INPUT + in_interface: internal + protocol: tcp + destination_port: "5001" + jump: ACCEPT - - name: allow wireguard input traffic (internal) - iptables: - chain: INPUT - in_interface: eth0 - destination_port: "51821" - protocol: udp - jump: ACCEPT +- name: Allow wireguard input traffic (internal) + iptables: + chain: INPUT + in_interface: eth0 + destination_port: "51821" + protocol: udp + jump: ACCEPT - - name: persist iptables config - include_role: - name: "iptables-persistent" +- name: Persist iptables config + include_role: + name: "iptables_persistent" - - name: Create /services/data - ansible.builtin.file: - path: /services/data - state: directory - mode: '0755' +- name: Create /services/data + ansible.builtin.file: + path: /services/data + state: directory + mode: "0755" diff --git a/ansible/roles/enomoloch/meta/main.yml b/ansible/roles/enomoloch/meta/main.yml deleted file mode 100644 index 05bcbf4..0000000 --- a/ansible/roles/enomoloch/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: -- role: "docker" diff --git a/ansible/roles/enomoloch/tasks/main.yml b/ansible/roles/enomoloch/tasks/main.yml deleted file mode 100644 index 00ccb83..0000000 --- a/ansible/roles/enomoloch/tasks/main.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- - - name: Increase max_map_count - sysctl: - name: vm.max_map_count - value: "262144" - sysctl_set: yes - - - name: Install tcpdump - apt: - name: tcpdump - state: present - update_cache: yes - - - name: Copy over the enodump unit - copy: - src: enodump.service - dest: /etc/systemd/system/enodump.service - changed_when: false - - - name: Disable tcpdump apparmor profile - file: - path: /etc/apparmor.d/disable/usr.bin.tcpdump - src: /etc/apparmor.d/usr.bin.tcpdump - state: link - force: yes - follow: no - - - name: Ensure /services/EnoMoloch exists - file: - path: /services/EnoMoloch - state: directory - - - name: Copy over the EnoMoloch docker-compose file - copy: - src: docker-compose.yml - dest: /services/EnoMoloch/docker-compose.yml - changed_when: false - - - name: Reload system daemon - systemd: - name: "enodump" # ansible <2.4 always requires 'name' - daemon_reload: yes - - - name: Ensure /pcaps exists - file: - path: /pcaps - state: directory - owner: tcpdump - group: tcpdump - - - name: Enable and start enodump - service: - name: enodump - enabled: yes - - - name: "Block access to moloch from game interface" - iptables: - chain: DOCKER-USER - in_interface: game - protocol: tcp - source: "10.13.0.0/24" - destination_port: "8005" - jump: DROP - - - name: persist iptables config - include_role: - name: "iptables-persistent" diff --git a/ansible/roles/filebeat/tasks/main.yml b/ansible/roles/filebeat/tasks/main.yml index f55a3c1..4ce0dd4 100644 --- a/ansible/roles/filebeat/tasks/main.yml +++ b/ansible/roles/filebeat/tasks/main.yml @@ -1,29 +1,31 @@ --- - - name: Check if Filebeat is installed - command: dpkg-query -W filebeat - register: filebeat_check_deb - failed_when: filebeat_check_deb.rc > 1 - changed_when: filebeat_check_deb.rc == 1 +- name: Check if Filebeat is installed + command: dpkg-query -W filebeat + register: filebeat_check_deb + failed_when: filebeat_check_deb.rc > 1 + changed_when: filebeat_check_deb.rc == 1 - - name: Download Filebeat - get_url: - url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.4.3-amd64.deb" - dest: "/tmp/filebeat.deb" - when: filebeat_check_deb.rc == 1 +- name: Download Filebeat + get_url: + url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.4.3-amd64.deb" + dest: "/tmp/filebeat.deb" + mode: "0644" + when: filebeat_check_deb.rc == 1 - - name: Install Filebeat - apt: - deb: "/tmp/filebeat.deb" - become: yes - when: filebeat_check_deb.rc == 1 +- name: Install Filebeat + apt: + deb: "/tmp/filebeat.deb" + become: true + when: filebeat_check_deb.rc == 1 - - name: Copy filebeat.yml to host - template: - src: "filebeat.yml.j2" - dest: "/etc/filebeat/filebeat.yml" +- name: Copy filebeat.yml to host + template: + src: "filebeat.yml.j2" + dest: "/etc/filebeat/filebeat.yml" + mode: "0644" - - name: Enable and start Filebeat - service: - name: filebeat - state: restarted - enabled: yes +- name: Enable and start Filebeat + service: + name: filebeat + state: restarted + enabled: true diff --git a/ansible/roles/firewall/tasks/main.yml b/ansible/roles/firewall/tasks/main.yml index 62f89a2..6b196f6 100644 --- a/ansible/roles/firewall/tasks/main.yml +++ b/ansible/roles/firewall/tasks/main.yml @@ -1,105 +1,120 @@ --- - - name: allow all loopback traffic - iptables: - chain: INPUT - in_interface: lo - jump: ACCEPT +- name: Allow all loopback traffic + iptables: + chain: INPUT + in_interface: lo + jump: ACCEPT - - name: allow related/established input traffic - iptables: - chain: INPUT - ctstate: [RELATED, ESTABLISHED] - jump: ACCEPT +- name: Allow related/established input traffic + iptables: + chain: INPUT + ctstate: [RELATED, ESTABLISHED] + jump: ACCEPT - - name: allow SSH traffic - iptables: - chain: INPUT - protocol: tcp - destination_port: "22" - ctstate: NEW - syn: match - jump: ACCEPT +- name: Allow SSH traffic + iptables: + chain: INPUT + protocol: tcp + destination_port: "22" + ctstate: NEW + syn: match + jump: ACCEPT - - name: allow ICMP ping requests - iptables: - chain: INPUT - protocol: icmp - icmp_type: "8" - jump: ACCEPT +- name: Allow ICMP ping requests + iptables: + chain: INPUT + protocol: icmp + icmp_type: "8" + jump: ACCEPT - - name: drop all input by default - iptables: - chain: INPUT - policy: DROP +- name: Drop all input by default + iptables: + chain: INPUT + policy: DROP - - name: drop all forward by default - iptables: - chain: FORWARD - policy: DROP +- name: Drop all forward by default + iptables: + chain: FORWARD + policy: DROP - - name: allow IPv6 loopback traffic - iptables: - chain: INPUT - in_interface: lo - jump: ACCEPT - ip_version: ipv6 +- name: Allow IPv6 loopback traffic + iptables: + chain: INPUT + in_interface: lo + jump: ACCEPT + ip_version: ipv6 - - name: allow IPv6 related/established input traffic - iptables: - chain: INPUT - ctstate: [RELATED, ESTABLISHED] - jump: ACCEPT - ip_version: ipv6 +- name: Allow IPv6 related/established input traffic + iptables: + chain: INPUT + ctstate: [RELATED, ESTABLISHED] + jump: ACCEPT + ip_version: ipv6 - - name: allow SSH traffic - iptables: - chain: INPUT - protocol: tcp - destination_port: "22" - ctstate: NEW - syn: match - jump: ACCEPT - ip_version: ipv6 +- name: Allow SSH traffic + iptables: + chain: INPUT + protocol: tcp + destination_port: "22" + ctstate: NEW + syn: match + jump: ACCEPT + ip_version: ipv6 - - name: permit needed ICMPv6 packet types (IPv6) - iptables: - chain: INPUT - protocol: ipv6-icmp - icmp_type: "{{ item }}" - jump: ACCEPT - ip_version: ipv6 - loop: ["1", "2", "3", "4", "133", "134", "135", "136", "137", "141", "142", "148", "149"] +- name: Permit needed ICMPv6 packet types (IPv6) + iptables: + chain: INPUT + protocol: ipv6-icmp + icmp_type: "{{ item }}" + jump: ACCEPT + ip_version: ipv6 + loop: + [ + "1", + "2", + "3", + "4", + "133", + "134", + "135", + "136", + "137", + "141", + "142", + "148", + "149", + ] - - name: permit needed ICMPv6 packet types from local addresses (IPv6) - iptables: - chain: INPUT - protocol: ipv6-icmp - icmp_type: "{{ item }}" - source: fe80::/10 - jump: ACCEPT - ip_version: ipv6 - loop: ["130", "131", "132", "143", "151", "152", "153"] +- name: Permit needed ICMPv6 packet types from local addresses (IPv6) + iptables: + chain: INPUT + protocol: ipv6-icmp + icmp_type: "{{ item }}" + source: fe80::/10 + jump: ACCEPT + ip_version: ipv6 + loop: ["130", "131", "132", "143", "151", "152", "153"] - - name: allow ICMPv6 ping requests (IPv6) - iptables: - chain: INPUT - protocol: ipv6-icmp - icmp_type: "128" - jump: ACCEPT - ip_version: ipv6 +- name: Allow ICMPv6 ping requests (IPv6) + iptables: + chain: INPUT + protocol: ipv6-icmp + icmp_type: "128" + jump: ACCEPT + ip_version: ipv6 - - name: disable IPv6 input - iptables: - chain: INPUT - policy: DROP - ip_version: ipv6 +- name: Disable IPv6 input + iptables: + chain: INPUT + policy: DROP + ip_version: ipv6 - - name: disable IPv6 forward - iptables: - chain: FORWARD - policy: DROP - ip_version: ipv6 +- name: Disable IPv6 forward + iptables: + chain: FORWARD + policy: DROP + ip_version: ipv6 - - name: persist iptables config - include_role: - name: "iptables-persistent" +- name: Persist iptables config + include_role: + name: "iptables_persistent" diff --git a/ansible/roles/iptables-persistent/meta/main.yml b/ansible/roles/iptables-persistent/meta/main.yml deleted file mode 100644 index fc0e0b6..0000000 --- a/ansible/roles/iptables-persistent/meta/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -allow_duplicates: true -dependencies: -- role: "programs" - vars: - program_list: - - iptables-persistent diff --git a/ansible/roles/iptables-persistent/tasks/main.yml b/ansible/roles/iptables-persistent/tasks/main.yml deleted file mode 100644 index f7d4300..0000000 --- a/ansible/roles/iptables-persistent/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - - name: persist iptables config - shell: "iptables-save > /etc/iptables/rules.v4" - - - name: persist ip6tables config - shell: "ip6tables-save > /etc/iptables/rules.v6" diff --git a/ansible/roles/iptables_persistent/meta/main.yml b/ansible/roles/iptables_persistent/meta/main.yml new file mode 100644 index 0000000..5452ae7 --- /dev/null +++ b/ansible/roles/iptables_persistent/meta/main.yml @@ -0,0 +1,6 @@ +allow_duplicates: true +dependencies: + - role: "programs" + vars: + programs_list: + - iptables-persistent diff --git a/ansible/roles/iptables_persistent/tasks/main.yml b/ansible/roles/iptables_persistent/tasks/main.yml new file mode 100644 index 0000000..3ed5490 --- /dev/null +++ b/ansible/roles/iptables_persistent/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Persist iptables config + shell: "iptables-save > /etc/iptables/rules.v4" + +- name: Persist ip6tables config + shell: "ip6tables-save > /etc/iptables/rules.v6" diff --git a/ansible/roles/journalbeat/tasks/main.yml b/ansible/roles/journalbeat/tasks/main.yml index 97ebd5e..baf28ab 100644 --- a/ansible/roles/journalbeat/tasks/main.yml +++ b/ansible/roles/journalbeat/tasks/main.yml @@ -1,29 +1,31 @@ --- - - name: Check if journalbeat is installed - command: dpkg-query -W journalbeat - register: journalbeat_check_deb - failed_when: journalbeat_check_deb.rc > 1 - changed_when: journalbeat_check_deb.rc == 1 +- name: Check if journalbeat is installed + command: dpkg-query -W journalbeat + register: journalbeat_check_deb + failed_when: journalbeat_check_deb.rc > 1 + changed_when: journalbeat_check_deb.rc == 1 - - name: Download journalbeat - get_url: - url: "https://artifacts.elastic.co/downloads/beats/journalbeat/journalbeat-7.6.2-amd64.deb" - dest: "/tmp/journalbeat.deb" - when: journalbeat_check_deb.rc == 1 +- name: Download journalbeat + get_url: + url: "https://artifacts.elastic.co/downloads/beats/journalbeat/journalbeat-7.6.2-amd64.deb" + dest: "/tmp/journalbeat.deb" + mode: "0644" + when: journalbeat_check_deb.rc == 1 - - name: Install journalbeat - apt: - deb: "/tmp/journalbeat.deb" - become: yes - when: journalbeat_check_deb.rc == 1 +- name: Install journalbeat + apt: + deb: "/tmp/journalbeat.deb" + become: true + when: journalbeat_check_deb.rc == 1 - - name: Sync journalbeat config - template: - src: journalbeat.yml.j2 - dest: /etc/journalbeat/journalbeat.yml - - - name: Enable and start journalbeat - service: - name: journalbeat - state: restarted - enabled: yes +- name: Sync journalbeat config + template: + src: journalbeat.yml.j2 + dest: /etc/journalbeat/journalbeat.yml + mode: "0644" + +- name: Enable and start journalbeat + service: + name: journalbeat + state: restarted + enabled: true diff --git a/ansible/roles/metricbeat/tasks/main.yml b/ansible/roles/metricbeat/tasks/main.yml index 6358739..9facdf8 100644 --- a/ansible/roles/metricbeat/tasks/main.yml +++ b/ansible/roles/metricbeat/tasks/main.yml @@ -9,21 +9,23 @@ get_url: url: "https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-8.4.3-amd64.deb" dest: "/tmp/metricbeat.deb" + mode: "0644" when: metricbeat_check_deb.rc == 1 - name: Install Metricbeat apt: deb: "/tmp/metricbeat.deb" - become: yes + become: true when: metricbeat_check_deb.rc == 1 - name: Copy Metricbeat config template: src: metricbeat.yml.j2 dest: /etc/metricbeat/metricbeat.yml + mode: "0644" - name: Enable and start Metricbeat service: name: metricbeat state: restarted - enabled: yes + enabled: true diff --git a/ansible/roles/pip-packages/meta/main.yml b/ansible/roles/pip-packages/meta/main.yml deleted file mode 100644 index 87bbb4c..0000000 --- a/ansible/roles/pip-packages/meta/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -dependencies: -- role: "programs" - vars: - program_list: - - python3-pip diff --git a/ansible/roles/pip-packages/tasks/main.yml b/ansible/roles/pip-packages/tasks/main.yml deleted file mode 100644 index 9da92b2..0000000 --- a/ansible/roles/pip-packages/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - - name: Install pip packages - ansible.builtin.pip: - name: "{{ pip_list }}" - umask: "0022" diff --git a/ansible/roles/pip_packages/meta/main.yml b/ansible/roles/pip_packages/meta/main.yml new file mode 100644 index 0000000..b3dd763 --- /dev/null +++ b/ansible/roles/pip_packages/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: + - role: "programs" + vars: + programs_list: + - python3-pip diff --git a/ansible/roles/pip_packages/tasks/main.yml b/ansible/roles/pip_packages/tasks/main.yml new file mode 100644 index 0000000..05b687e --- /dev/null +++ b/ansible/roles/pip_packages/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: Install pip packages + ansible.builtin.pip: + name: "{{ pip_packages_list }}" + umask: "0022" diff --git a/ansible/roles/programs/tasks/main.yml b/ansible/roles/programs/tasks/main.yml index 53dedf4..2059444 100644 --- a/ansible/roles/programs/tasks/main.yml +++ b/ansible/roles/programs/tasks/main.yml @@ -1,8 +1,8 @@ --- - - name: Install all the needed programs and do an update - apt: - name: "{{ program_list }}" - force_apt_get: yes - autoremove: yes - cache_valid_time: 3000 - state: present +- name: Install all the needed programs and do an update + apt: + name: "{{ programs_list }}" + force_apt_get: true + autoremove: true + cache_valid_time: 3000 + state: present diff --git a/ansible/roles/root-password-login/tasks/main.yml b/ansible/roles/root_password_login/tasks/main.yml similarity index 91% rename from ansible/roles/root-password-login/tasks/main.yml rename to ansible/roles/root_password_login/tasks/main.yml index ac5f449..9fb6ab8 100644 --- a/ansible/roles/root-password-login/tasks/main.yml +++ b/ansible/roles/root_password_login/tasks/main.yml @@ -5,4 +5,4 @@ regexp: '^PermitRootLogin' line: "PermitRootLogin yes" state: present - backup: yes + backup: true diff --git a/ansible/roles/router-openvpn/meta/main.yml b/ansible/roles/router-openvpn/meta/main.yml deleted file mode 100644 index 1ab1c16..0000000 --- a/ansible/roles/router-openvpn/meta/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -dependencies: -- role: "programs" - vars: - program_list: - - openvpn diff --git a/ansible/roles/bambi-arkime/files/Dockerfile b/ansible/roles/router_arkime/files/Dockerfile similarity index 100% rename from ansible/roles/bambi-arkime/files/Dockerfile rename to ansible/roles/router_arkime/files/Dockerfile diff --git a/ansible/roles/bambi-arkime/files/arkime-capture.sh b/ansible/roles/router_arkime/files/arkime-capture.sh similarity index 100% rename from ansible/roles/bambi-arkime/files/arkime-capture.sh rename to ansible/roles/router_arkime/files/arkime-capture.sh diff --git a/ansible/roles/bambi-arkime/files/arkime-viewer.sh b/ansible/roles/router_arkime/files/arkime-viewer.sh similarity index 100% rename from ansible/roles/bambi-arkime/files/arkime-viewer.sh rename to ansible/roles/router_arkime/files/arkime-viewer.sh diff --git a/ansible/roles/bambi-arkime/files/config.ini b/ansible/roles/router_arkime/files/config.ini similarity index 100% rename from ansible/roles/bambi-arkime/files/config.ini rename to ansible/roles/router_arkime/files/config.ini diff --git a/ansible/roles/router_arkime/files/docker-compose.arkime.yml b/ansible/roles/router_arkime/files/docker-compose.arkime.yml new file mode 100644 index 0000000..fc10424 --- /dev/null +++ b/ansible/roles/router_arkime/files/docker-compose.arkime.yml @@ -0,0 +1,12 @@ +services: + arkime: + # image: ghcr.io/enoflag/enoarkime:nightly + build: . + restart: unless-stopped + ports: + - 8005:8005 + volumes: + - /pcaps_arkime:/opt/arkime/raw + - .:/BambiArkime:ro + - ./config.ini:/opt/arkime/etc/config.ini:ro + network_mode: host diff --git a/ansible/roles/router_arkime/files/docker-compose.elasticsearch.yml b/ansible/roles/router_arkime/files/docker-compose.elasticsearch.yml new file mode 100644 index 0000000..2ec00be --- /dev/null +++ b/ansible/roles/router_arkime/files/docker-compose.elasticsearch.yml @@ -0,0 +1,17 @@ +services: + elasticsearch: + restart: unless-stopped + image: elasticsearch:8.11.1 + environment: + - xpack.security.enabled=false + - node.name=[[ROUTER]] + - cluster.name=bambiarkime + - cluster.initial_master_nodes=[[INITIAL_MASTER_NODES]] + - discovery.seed_hosts=[[SEED_HOSTS]] + - network.publish_host=[[ROUTER]] + - http.publish_host=[[ROUTER]] + - transport.publish_host=[[ROUTER]] + ports: + - 9200:9200 + - 9300:9300 + network_mode: host diff --git a/ansible/roles/bambi-arkime/files/docker-entrypoint.sh b/ansible/roles/router_arkime/files/docker-entrypoint.sh similarity index 100% rename from ansible/roles/bambi-arkime/files/docker-entrypoint.sh rename to ansible/roles/router_arkime/files/docker-entrypoint.sh diff --git a/ansible/roles/router_arkime/meta/main.yml b/ansible/roles/router_arkime/meta/main.yml new file mode 100644 index 0000000..227362a --- /dev/null +++ b/ansible/roles/router_arkime/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - role: "docker" + - role: "ssh_keys" diff --git a/ansible/roles/bambi-arkime/tasks/main.yml b/ansible/roles/router_arkime/tasks/main.yml similarity index 76% rename from ansible/roles/bambi-arkime/tasks/main.yml rename to ansible/roles/router_arkime/tasks/main.yml index 8f08a26..38f2caa 100644 --- a/ansible/roles/bambi-arkime/tasks/main.yml +++ b/ansible/roles/router_arkime/tasks/main.yml @@ -1,5 +1,5 @@ --- -- name: allow wireguard input traffic (internal) +- name: Allow wireguard input traffic (internal) iptables: chain: INPUT in_interface: eth0 @@ -8,32 +8,34 @@ jump: ACCEPT # TODO fixme @ldruschk -- shell: iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT -- shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 9300 -j ACCEPT -- shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 8005 -j ACCEPT +- shell: iptables -A INPUT -i br-+ -p tcp -m tcp --dport 9200 -j ACCEPT # noqa: command-instead-of-shell +- shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 9300 -j ACCEPT # noqa: command-instead-of-shell +- shell: iptables -A INPUT -i internal -p tcp -m tcp --dport 8005 -j ACCEPT # noqa: command-instead-of-shell -- name: persist iptables config +- name: Persist iptables config include_role: - name: "iptables-persistent" + name: "iptables_persistent" - name: Increase max_map_count sysctl: name: vm.max_map_count value: "262144" - sysctl_set: yes + sysctl_set: true - name: Create /services/BambiArkimeElasticsearch file: path: /services/BambiArkimeElasticsearch state: directory + mode: "0755" -- name: docker-compose.elasticsearch.yml +- name: Copy docker-compose.elasticsearch.yml copy: src: docker-compose.elasticsearch.yml dest: /services/BambiArkimeElasticsearch/docker-compose.yml + mode: "0644" - name: Pull ES - shell: docker compose pull + shell: docker compose pull # noqa: command-instead-of-shell args: chdir: /services/BambiArkimeElasticsearch @@ -41,40 +43,43 @@ file: path: /services/BambiArkime state: directory + mode: "0755" - name: Copy docker-compose.arkime.yml copy: src: docker-compose.arkime.yml dest: /services/BambiArkime/docker-compose.yml + mode: "0644" - name: Copy config.ini copy: src: config.ini dest: /services/BambiArkime/config.ini + mode: "0644" - name: Copy Dockerfile copy: src: Dockerfile dest: /services/BambiArkime/Dockerfile + mode: "0644" - name: Copy arkime-capture.sh copy: src: arkime-capture.sh dest: /services/BambiArkime/arkime-capture.sh - mode: '755' + mode: "0755" - name: Copy arkime-viewer.sh copy: src: arkime-viewer.sh dest: /services/BambiArkime/arkime-viewer.sh - mode: '755' + mode: "0755" - name: Copy docker-entrypoint.sh copy: src: docker-entrypoint.sh dest: /services/BambiArkime/docker-entrypoint.sh - mode: '755' - + mode: "0755" # TODO once wait for elasticsearch green works # - name: build # shell: "docker compose build" diff --git a/ansible/roles/router_iptables/meta/main.yml b/ansible/roles/router_iptables/meta/main.yml new file mode 100644 index 0000000..87d7865 --- /dev/null +++ b/ansible/roles/router_iptables/meta/main.yml @@ -0,0 +1,7 @@ +dependencies: + - role: "wireguard" + - role: "firewall" + - role: "programs" + vars: + programs_list: + - iptables-persistent diff --git a/ansible/roles/router_iptables/tasks/main.yml b/ansible/roles/router_iptables/tasks/main.yml new file mode 100644 index 0000000..2b48f49 --- /dev/null +++ b/ansible/roles/router_iptables/tasks/main.yml @@ -0,0 +1,154 @@ +--- +- name: Allow wireguard input traffic (game) + iptables: + chain: INPUT + in_interface: eth0 + destination_port: "51820" + protocol: udp + jump: ACCEPT + +- name: Allow wireguard input traffic (internal) + iptables: + chain: INPUT + in_interface: eth0 + destination_port: "51821" + protocol: udp + jump: ACCEPT + +- name: "Enable ipv4 forward" + sysctl: + name: net.ipv4.ip_forward + value: "1" + sysctl_set: true + +- name: Allow related/established forward traffic + iptables: + chain: FORWARD + ctstate: [RELATED, ESTABLISHED] + jump: ACCEPT + +- name: "Capture iptables-save output" + shell: "iptables-save" + register: iptablessave + +- name: "Insert limit bw internal" + shell: "iptables -I FORWARD -i router -o internal -m hashlimit --hashlimit-above 10mb/s --hashlimit-burst 20mb --hashlimit-mode srcip --hashlimit-name bwi --hashlimit-srcmask 24 -j DROP" + when: '"-A FORWARD -i router -o internal -m hashlimit --hashlimit-above 10mb/s --hashlimit-burst 20mb --hashlimit-mode srcip --hashlimit-name bwi --hashlimit-srcmask 24 -j DROP" not in iptablessave.stdout' + +- name: "Insert limit bw other" + shell: "iptables -I FORWARD -i router ! -o internal -m hashlimit --hashlimit-above 5mb/s --hashlimit-burst 10mb --hashlimit-mode srcip --hashlimit-name bw --hashlimit-srcmask 24 -j DROP" + when: '"-A FORWARD -i router ! -o internal -m hashlimit --hashlimit-above 5mb/s --hashlimit-burst 10mb --hashlimit-mode srcip --hashlimit-name bw --hashlimit-srcmask 24 -j DROP" not in iptablessave.stdout' + +- name: "Insert limit conn internal" + shell: "iptables -I FORWARD -i router -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 2/sec --hashlimit-burst 10 --hashlimit-mode srcip,dstip --hashlimit-name subnewconns -j DROP" + when: '"-A FORWARD -i router -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 2/sec --hashlimit-burst 10 --hashlimit-mode srcip,dstip --hashlimit-name subnewconns -j DROP" not in iptablessave.stdout' + +- name: "Insert limit conn other" + shell: "iptables -I FORWARD -i router ! -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 50/sec --hashlimit-burst 100 --hashlimit-mode srcip,dstip --hashlimit-name newconns -j DROP" + when: '"-A FORWARD -i router ! -o internal -m conntrack --ctstate NEW -m hashlimit --hashlimit-above 50/sec --hashlimit-burst 100 --hashlimit-mode srcip,dstip --hashlimit-name newconns -j DROP" not in iptablessave.stdout' + +- name: Allow related/established forward traffic + iptables: + chain: FORWARD + ctstate: [RELATED, ESTABLISHED] + jump: ACCEPT + +- name: Allow all internal traffic + iptables: + chain: FORWARD + in_interface: internal + out_interface: internal + jump: ACCEPT + +# this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow +- name: Prevent masquerading for intra-team traffic part + shell: "iptables -t nat -A POSTROUTING -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j RETURN" + when: '"-A POSTROUTING -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j RETURN" not in iptablessave.stdout' + with_sequence: start=1 end=255 + +- name: Masquerade outgoing traffic on the router interface + iptables: + table: nat + chain: POSTROUTING + out_interface: router + jump: MASQUERADE + +- name: DNAT the flag submission IP to the engine + iptables: + table: nat + chain: PREROUTING + destination: 10.0.13.37 + jump: DNAT + to_destination: 192.168.1.0 + +- name: Allow pinging the flag submission endpoint + iptables: + chain: FORWARD + in_interface: router + out_interface: internal + destination: 192.168.1.0 + protocol: icmp + icmp_type: "8" + jump: ACCEPT + +- name: Allow flag submission traffic from vulnboxes + iptables: + chain: FORWARD + in_interface: router + out_interface: internal + destination: 192.168.1.0 + protocol: tcp + destination_port: "1337" + jump: ACCEPT + +- name: Allow flag submission traffic from vpn connections + iptables: + chain: FORWARD + in_interface: team+ + out_interface: internal + destination: 192.168.1.0 + protocol: tcp + destination_port: "1337" + jump: ACCEPT + +- name: Allow enoctfportal traffic from vulnboxes + iptables: + chain: FORWARD + in_interface: router + out_interface: internal + destination: 192.168.1.0 + protocol: tcp + destination_port: "5001" + jump: ACCEPT + +- name: Allow enoctfportal traffic from vpn connections + iptables: + chain: FORWARD + in_interface: team+ + out_interface: internal + destination: 192.168.1.0 + protocol: tcp + destination_port: "5001" + jump: ACCEPT + +- name: Allow traffic from the internal network to teams + iptables: + chain: FORWARD + in_interface: internal + out_interface: router + jump: ACCEPT + +# this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow +- name: Allow intra-team traffic part 1 + shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j ACCEPT" + when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' + with_sequence: start=1 end=255 + +- name: Allow intra-team traffic part 2 + shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o team+ -j ACCEPT" + when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' + with_sequence: start=1 end=255 + +- name: Persist iptables config + include_role: + name: "iptables_persistent" diff --git a/ansible/roles/router_openvpn/meta/main.yml b/ansible/roles/router_openvpn/meta/main.yml new file mode 100644 index 0000000..90d24ba --- /dev/null +++ b/ansible/roles/router_openvpn/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: + - role: "programs" + vars: + programs_list: + - openvpn diff --git a/ansible/roles/router-openvpn/tasks/main.yml b/ansible/roles/router_openvpn/tasks/main.yml similarity index 61% rename from ansible/roles/router-openvpn/tasks/main.yml rename to ansible/roles/router_openvpn/tasks/main.yml index 8a10e7a..e6de117 100644 --- a/ansible/roles/router-openvpn/tasks/main.yml +++ b/ansible/roles/router_openvpn/tasks/main.yml @@ -1,10 +1,11 @@ --- -- name: copy team openvpn server configs +- name: Copy team openvpn server configs copy: src: "{{ playbook_dir }}/../config/export/ansible/routers/openvpn/" dest: /etc/openvpn/server + mode: "0644" -- name: allow openvpn input traffic +- name: Allow openvpn input traffic iptables: chain: INPUT in_interface: eth0 @@ -12,6 +13,6 @@ protocol: udp jump: ACCEPT -- name: persist iptables config +- name: Persist iptables config include_role: - name: "iptables-persistent" + name: "iptables_persistent" diff --git a/ansible/roles/router_trafficcapture/tasks/main.yml b/ansible/roles/router_trafficcapture/tasks/main.yml index b13eddc..b5fbf8d 100644 --- a/ansible/roles/router_trafficcapture/tasks/main.yml +++ b/ansible/roles/router_trafficcapture/tasks/main.yml @@ -5,6 +5,7 @@ state: directory owner: tcpdump group: tcpdump + mode: "0755" - name: Ensure /pcaps_arkime exists file: @@ -12,6 +13,7 @@ state: directory owner: tcpdump group: tcpdump + mode: "0755" - name: Copy move_pcap.sh copy: @@ -19,33 +21,34 @@ dest: /pcaps/move_pcap.sh owner: tcpdump group: tcpdump - mode: '0700' + mode: "0700" - name: Install tcpdump apt: name: tcpdump state: present - update_cache: yes + update_cache: true - name: Copy over the enorouterdump unit copy: src: enorouterdump.service dest: /etc/systemd/system/enorouterdump.service + mode: "0644" - name: Disable tcpdump apparmor profile file: path: /etc/apparmor.d/disable/usr.bin.tcpdump src: /etc/apparmor.d/usr.bin.tcpdump state: link - force: yes - follow: no + force: true + follow: false - name: Reload system daemon systemd: name: "enorouterdump" # ansible <2.4 always requires 'name' - daemon_reload: yes + daemon_reload: true - name: Enable enorouterdump service: name: enorouterdump - enabled: yes + enabled: true diff --git a/ansible/roles/ssh_keys/tasks/main.yml b/ansible/roles/ssh_keys/tasks/main.yml new file mode 100644 index 0000000..8f9b32e --- /dev/null +++ b/ansible/roles/ssh_keys/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Pull authkeys from github + authorized_key: + user: root + state: present + key: "https://github.com/{{ item }}.keys" + with_items: "{{ github_ssh_keys }}" diff --git a/ansible/roles/tmux/meta/main.yml b/ansible/roles/tmux/meta/main.yml index 1ed982e..d264104 100644 --- a/ansible/roles/tmux/meta/main.yml +++ b/ansible/roles/tmux/meta/main.yml @@ -1,5 +1,5 @@ dependencies: -- role: "programs" - vars: - program_list: - - tmux + - role: "programs" + vars: + programs_list: + - tmux diff --git a/ansible/roles/tmux/tasks/main.yml b/ansible/roles/tmux/tasks/main.yml index 892f90e..82fb280 100644 --- a/ansible/roles/tmux/tasks/main.yml +++ b/ansible/roles/tmux/tasks/main.yml @@ -1,5 +1,6 @@ --- - - name: Copy tmux conf - copy: - src: .tmux.conf - dest: /root/.tmux.conf +- name: Copy tmux conf + copy: + src: .tmux.conf + dest: /root/.tmux.conf + mode: "0644" diff --git a/ansible/roles/vuln_checkers/meta/main.yml b/ansible/roles/vuln_checkers/meta/main.yml index ddbfc7d..faa693c 100644 --- a/ansible/roles/vuln_checkers/meta/main.yml +++ b/ansible/roles/vuln_checkers/meta/main.yml @@ -1,7 +1,7 @@ dependencies: -- role: "docker" -- role: "programs" - vars: - program_list: - - git -- role: "bambi-ssh-keys" + - role: "docker" + - role: "programs" + vars: + programs_list: + - git + - role: "ssh_keys" diff --git a/ansible/roles/vuln_checkers/tasks/main.yml b/ansible/roles/vuln_checkers/tasks/main.yml index 2518ff6..28d1673 100644 --- a/ansible/roles/vuln_checkers/tasks/main.yml +++ b/ansible/roles/vuln_checkers/tasks/main.yml @@ -1,35 +1,35 @@ --- - - name: Create /services - file: - path: /services - state: directory +- name: Create /services + file: + path: /services + state: directory + mode: "0755" - - name: Clean local checker services cache - become: no - local_action: - module: file - path: ./services/{{ inventory_hostname }}/{{ item.key }} - state: absent - with_dict: "{{ vulnerable_services }}" +- name: Clean local checker services cache + file: + path: ./services/{{ inventory_hostname }}/{{ item.key }} + state: absent + with_dict: "{{ vulnerable_services }}" + delegate_to: localhost - - name: Clone checkers services locally - become: no - local_action: - module: git - repo: "{{ item.value }}" - dest: ./services/{{ inventory_hostname }}/{{ item.key }} - accept_hostkey: yes - key_file: "{{ playbook_dir }}/../id_ed25519" - with_dict: "{{ vulnerable_services }}" +- name: Clone checkers services locally + become: false + git: # noqa: latest + repo: "{{ item.value }}" + dest: ./services/{{ inventory_hostname }}/{{ item.key }} + accept_hostkey: true + key_file: "{{ playbook_dir }}/../id_ed25519" + with_dict: "{{ vulnerable_services }}" + delegate_to: localhost - - name: Copy checkers to gameserver - synchronize: - src: ./services/{{ inventory_hostname }}/{{ item.key }}/checker/ - dest: /services/checker_{{ item.key }} - with_dict: "{{ vulnerable_services }}" +- name: Copy checkers to gameserver + synchronize: + src: ./services/{{ inventory_hostname }}/{{ item.key }}/checker/ + dest: /services/checker_{{ item.key }} + with_dict: "{{ vulnerable_services }}" - - name: pull/build service - shell: docker compose pull && docker compose build - args: - chdir: /services/checker_{{ item.key }} - with_dict: "{{ vulnerable_services }}" +- name: Pull/build service + shell: docker compose pull && docker compose build + args: + chdir: /services/checker_{{ item.key }} + with_dict: "{{ vulnerable_services }}" diff --git a/ansible/roles/vuln_services/meta/main.yml b/ansible/roles/vuln_services/meta/main.yml index ddbfc7d..faa693c 100644 --- a/ansible/roles/vuln_services/meta/main.yml +++ b/ansible/roles/vuln_services/meta/main.yml @@ -1,7 +1,7 @@ dependencies: -- role: "docker" -- role: "programs" - vars: - program_list: - - git -- role: "bambi-ssh-keys" + - role: "docker" + - role: "programs" + vars: + programs_list: + - git + - role: "ssh_keys" diff --git a/ansible/roles/vuln_services/tasks/main.yml b/ansible/roles/vuln_services/tasks/main.yml index 8d808b7..ff505ba 100644 --- a/ansible/roles/vuln_services/tasks/main.yml +++ b/ansible/roles/vuln_services/tasks/main.yml @@ -1,34 +1,35 @@ --- - - name: Create /services - file: - path: /services - state: directory +- name: Create /services + file: + path: /services + state: directory + mode: "0755" - - name: Clean local services cache - become: no - local_action: - module: file - path: ./services/{{ inventory_hostname }}/dockered_vulnerable_services - state: absent +- name: Clean local services cache + become: false + file: + path: ./services/{{ inventory_hostname }}/dockered_vulnerable_services + state: absent + delegate_to: localhost - - name: Clone services locally - become: no - local_action: - module: git - repo: "{{ item.value }}" - dest: ./services/{{ inventory_hostname }}/dockered_vulnerable_services/{{ item.key }} - key_file: "{{ playbook_dir }}/../id_ed25519" - accept_hostkey: yes - with_dict: "{{ vulnerable_services }}" +- name: Clone services locally + become: false + git: # noqa: latest + repo: "{{ item.value }}" + dest: ./services/{{ inventory_hostname }}/dockered_vulnerable_services/{{ item.key }} + key_file: "{{ playbook_dir }}/../id_ed25519" + accept_hostkey: true + with_dict: "{{ vulnerable_services }}" + delegate_to: localhost - - name: Copy services to vulnbox - synchronize: - src: ./services/{{ inventory_hostname }}/dockered_vulnerable_services/{{ item.key }}/service/ - dest: /services/{{ item.key }} - with_dict: "{{ vulnerable_services }}" +- name: Copy services to vulnbox + synchronize: + src: ./services/{{ inventory_hostname }}/dockered_vulnerable_services/{{ item.key }}/service/ + dest: /services/{{ item.key }} + with_dict: "{{ vulnerable_services }}" - - name: pull/build service - shell: docker compose pull && docker compose build - args: - chdir: /services/{{ item.key }} - with_dict: "{{ vulnerable_services }}" +- name: Pull/build service + shell: docker compose pull && docker compose build + args: + chdir: /services/{{ item.key }} + with_dict: "{{ vulnerable_services }}" diff --git a/ansible/roles/enomoloch/files/docker-compose.yml b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml similarity index 100% rename from ansible/roles/enomoloch/files/docker-compose.yml rename to ansible/roles/vulnbox_enomoloch/files/docker-compose.yml diff --git a/ansible/roles/enomoloch/files/enodump.service b/ansible/roles/vulnbox_enomoloch/files/enodump.service similarity index 100% rename from ansible/roles/enomoloch/files/enodump.service rename to ansible/roles/vulnbox_enomoloch/files/enodump.service diff --git a/ansible/roles/vulnbox_enomoloch/meta/main.yml b/ansible/roles/vulnbox_enomoloch/meta/main.yml new file mode 100644 index 0000000..15afd61 --- /dev/null +++ b/ansible/roles/vulnbox_enomoloch/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - role: "docker" diff --git a/ansible/roles/vulnbox_enomoloch/tasks/main.yml b/ansible/roles/vulnbox_enomoloch/tasks/main.yml new file mode 100644 index 0000000..88e4783 --- /dev/null +++ b/ansible/roles/vulnbox_enomoloch/tasks/main.yml @@ -0,0 +1,71 @@ +--- +- name: Increase max_map_count + sysctl: + name: vm.max_map_count + value: "262144" + sysctl_set: true + +- name: Install tcpdump + apt: + name: tcpdump + state: present + update_cache: true + +- name: Copy over the enodump unit + copy: + src: enodump.service + dest: /etc/systemd/system/enodump.service + mode: "0644" + changed_when: false + +- name: Disable tcpdump apparmor profile + file: + path: /etc/apparmor.d/disable/usr.bin.tcpdump + src: /etc/apparmor.d/usr.bin.tcpdump + state: link + force: true + follow: false + +- name: Ensure /services/EnoMoloch exists + file: + path: /services/EnoMoloch + state: directory + mode: "0755" + +- name: Copy over the EnoMoloch docker-compose file + copy: + src: docker-compose.yml + dest: /services/EnoMoloch/docker-compose.yml + mode: "0644" + changed_when: false + +- name: Reload system daemon + systemd: + name: "enodump" # ansible <2.4 always requires 'name' + daemon_reload: true + +- name: Ensure /pcaps exists + file: + path: /pcaps + state: directory + owner: tcpdump + group: tcpdump + mode: "0644" + +- name: Enable and start enodump + service: + name: enodump + enabled: true + +- name: "Block access to moloch from game interface" + iptables: + chain: DOCKER-USER + in_interface: game + protocol: tcp + source: "10.13.0.0/24" + destination_port: "8005" + jump: DROP + +- name: Persist iptables config + include_role: + name: "iptables_persistent" diff --git a/ansible/roles/wireguard/meta/main.yml b/ansible/roles/wireguard/meta/main.yml index 161b64b..a7045d5 100644 --- a/ansible/roles/wireguard/meta/main.yml +++ b/ansible/roles/wireguard/meta/main.yml @@ -1,5 +1,5 @@ dependencies: -- role: "programs" - vars: - program_list: - - wireguard + - role: "programs" + vars: + programs_list: + - wireguard diff --git a/ansible/roles/wireguard/tasks/main.yml b/ansible/roles/wireguard/tasks/main.yml index 667caae..0ae44c9 100644 --- a/ansible/roles/wireguard/tasks/main.yml +++ b/ansible/roles/wireguard/tasks/main.yml @@ -1,6 +1,6 @@ --- - - name: Ensure wg directory exists - file: - path: /etc/wireguard - state: directory - mode: "700" +- name: Ensure wg directory exists + file: + path: /etc/wireguard + state: directory + mode: "700" diff --git a/ansible/roles/wireguard_configs/meta/main.yml b/ansible/roles/wireguard_configs/meta/main.yml index 656bcf7..1eeeafc 100644 --- a/ansible/roles/wireguard_configs/meta/main.yml +++ b/ansible/roles/wireguard_configs/meta/main.yml @@ -1,2 +1,2 @@ dependencies: -- role: "wireguard" + - role: "wireguard" diff --git a/ansible/roles/wireguard_configs/tasks/main.yml b/ansible/roles/wireguard_configs/tasks/main.yml index 27994c4..441932c 100644 --- a/ansible/roles/wireguard_configs/tasks/main.yml +++ b/ansible/roles/wireguard_configs/tasks/main.yml @@ -1,5 +1,6 @@ --- - - name: Copy wg configs - copy: - src: "{{ playbook_dir }}/../config/export/ansible/{{ config_dir }}/" - dest: /etc/wireguard/ +- name: Copy wg configs + copy: + src: "{{ playbook_dir }}/../config/export/ansible/{{ wireguard_configs_dir }}/" + dest: /etc/wireguard/ + mode: "0600" From d0ff1001f5d660b943334c64aa219f3e1255f1ab Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Tue, 9 Jul 2024 20:47:58 +0200 Subject: [PATCH 49/61] make network open/intra-team-traffic rules time-based --- ansible/config_bambi.yml.sample | 3 +++ ansible/roles/router_iptables/tasks/main.yml | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ansible/config_bambi.yml.sample b/ansible/config_bambi.yml.sample index 0b17b77..c35f54b 100644 --- a/ansible/config_bambi.yml.sample +++ b/ansible/config_bambi.yml.sample @@ -3,3 +3,6 @@ vulnerable_services: github_ssh_keys: [ ldruschk, ] +vulnbox_access_time: "2024-07-09T20:00:00" +network_open_time: "2024-07-09T21:00:00" +network_close_time: "2024-07-09T22:00:00" diff --git a/ansible/roles/router_iptables/tasks/main.yml b/ansible/roles/router_iptables/tasks/main.yml index 2b48f49..3d57c8f 100644 --- a/ansible/roles/router_iptables/tasks/main.yml +++ b/ansible/roles/router_iptables/tasks/main.yml @@ -139,16 +139,19 @@ jump: ACCEPT # this is a pretty hacky workaround, but the execution of the iptables-module in ansible is just painfully slow -- name: Allow intra-team traffic part 1 - shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -j ACCEPT" - when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' +- name: Allow intra-team traffic part 1 (time-based) + shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o router -m time --datestart {{ vulnbox_access_time }} -j ACCEPT" + # when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' # TODO: this is broken, needs to be revisited with_sequence: start=1 end=255 -- name: Allow intra-team traffic part 2 - shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o team+ -j ACCEPT" - when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' +- name: Allow intra-team traffic part 2 (time-based) + shell: "iptables -A FORWARD -s 10.1.{{ item }}.0/24 -d 10.1.{{ item }}.0/24 -o team+ -m time --datestart {{ vulnbox_access_time }} -j ACCEPT" + # when: '"-A FORWARD -s 10.1."+item+".0/24 -d 10.1."+item+".0/24 -o router -j ACCEPT" not in iptablessave.stdout' # TODO: this is broken, needs to be revisited with_sequence: start=1 end=255 +- name: Open game network (time-based) + shell: "iptables -A FORWARD -o router -m time --datestart {{ network_open_time }} --datestop {{ network_close_time }} -j ACCEPT" + - name: Persist iptables config include_role: name: "iptables_persistent" From 44d998f38c408d22094f06002001b406444b36d2 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Tue, 9 Jul 2024 20:54:59 +0200 Subject: [PATCH 50/61] make enodump depend on wg-quick (closes #46) --- ansible/roles/vulnbox_enomoloch/files/enodump.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/vulnbox_enomoloch/files/enodump.service b/ansible/roles/vulnbox_enomoloch/files/enodump.service index 9da632b..129f7c9 100644 --- a/ansible/roles/vulnbox_enomoloch/files/enodump.service +++ b/ansible/roles/vulnbox_enomoloch/files/enodump.service @@ -1,7 +1,7 @@ [Unit] Description=enodump packet capture -Requires=docker.service -After=docker.service containerd.service +Requires=docker.service wg-quick@game.service +After=docker.service containerd.service wg-quick@game.service [Service] Restart=always From 7ee4ca7648ca776d5556f686ccbf6a6bc86a093e Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Tue, 9 Jul 2024 21:03:50 +0200 Subject: [PATCH 51/61] update enoarkime on vulnboxes (closes #37) --- ansible/roles/vulnbox_enomoloch/files/docker-compose.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml index 57efc81..b9445ce 100644 --- a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml +++ b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml @@ -1,14 +1,12 @@ version: '3' services: moloch: - image: ghcr.io/enoflag/enomoloch:latest + image: ghcr.io/enoflag/enoarkime:latest ports: - 8005:8005 volumes: - "/pcaps:/data/moloch/raw" - environment: - - "MOLOCH_PASSWORD=moloch" elasticsearchmoloch: - image: elasticsearch:7.6.0 + image: elasticsearch:7.14.2 environment: - discovery.type=single-node From 58b70f4bcff9e3051ef11e4765f2842d747939db Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Wed, 10 Jul 2024 21:35:44 +0200 Subject: [PATCH 52/61] fix arkime on vulnbox --- ansible/roles/vulnbox_enomoloch/files/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml index b9445ce..ed09437 100644 --- a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml +++ b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml @@ -1,12 +1,12 @@ version: '3' services: moloch: - image: ghcr.io/enoflag/enoarkime:latest + image: ghcr.io/enoflag/enoarkime:5.1.0 ports: - 8005:8005 volumes: - - "/pcaps:/data/moloch/raw" - elasticsearchmoloch: + - "/pcaps:/opt/arkime/raw" + elasticsearch: image: elasticsearch:7.14.2 environment: - discovery.type=single-node From f9edd5c57bd5b25d8d442cb66927444599ee8f47 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Wed, 10 Jul 2024 21:35:49 +0200 Subject: [PATCH 53/61] fix endump systemd unit --- ansible/roles/vulnbox_enomoloch/files/enodump.service | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/vulnbox_enomoloch/files/enodump.service b/ansible/roles/vulnbox_enomoloch/files/enodump.service index 129f7c9..3a742e0 100644 --- a/ansible/roles/vulnbox_enomoloch/files/enodump.service +++ b/ansible/roles/vulnbox_enomoloch/files/enodump.service @@ -1,10 +1,12 @@ [Unit] Description=enodump packet capture -Requires=docker.service wg-quick@game.service +Requires=docker.service After=docker.service containerd.service wg-quick@game.service +StartLimitIntervalSec=0 [Service] Restart=always +RestartSec=3 TimeoutStartSec=300 WorkingDirectory=/pcaps From 122212283f57fca39d1dbacd0dbd28a29e79b36d Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Wed, 10 Jul 2024 21:49:12 +0200 Subject: [PATCH 54/61] bump enoarkime to 5.3.0 --- ansible/roles/router_arkime/files/Dockerfile | 2 +- ansible/roles/vulnbox_enomoloch/files/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/router_arkime/files/Dockerfile b/ansible/roles/router_arkime/files/Dockerfile index fea70dd..6a84b76 100644 --- a/ansible/roles/router_arkime/files/Dockerfile +++ b/ansible/roles/router_arkime/files/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/enoflag/enoarkime:5.1.0 +FROM ghcr.io/enoflag/enoarkime:5.3.0 COPY arkime-viewer.sh arkime-viewer.sh COPY arkime-capture.sh arkime-capture.sh diff --git a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml index ed09437..7e5d3da 100644 --- a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml +++ b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: moloch: - image: ghcr.io/enoflag/enoarkime:5.1.0 + image: ghcr.io/enoflag/enoarkime:5.3.0 ports: - 8005:8005 volumes: From 8499b2a63df2436970bca4d8088f94e219315bc6 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Fri, 12 Jul 2024 19:21:04 +0200 Subject: [PATCH 55/61] add proxy hosted on DO (untested/unfinished) --- terraform/.terraform.lock.hcl | 21 ++++++++ terraform/bambi.tf | 21 ++++---- terraform/bambichecker.tf | 14 +++--- terraform/bambivulnbox.tf | 16 ++++--- terraform/proxy.tf | 48 +++++++++++++++++++ terraform/squid_acl.txt | 79 +++++++++++++++++++++++++++++++ terraform/user_data_checker.tftpl | 14 ++++++ terraform/user_data_proxy.tftpl | 14 ++++++ terraform/user_data_vulnbox.tftpl | 14 ++++++ 9 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 terraform/proxy.tf create mode 100644 terraform/squid_acl.txt create mode 100644 terraform/user_data_proxy.tftpl diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 8d9c4ea..03461e5 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -38,3 +38,24 @@ provider "registry.terraform.io/timohirt/hetznerdns" { "zh:db998fe3bdcd4902e99fa470bb3f355883170cf4c711c8da0b5f1f4510f1be41", ] } + +provider "registry.terraform.io/vancluever/acme" { + version = "2.24.2" + constraints = "2.24.2" + hashes = [ + "h1:zxpXpKMgBbhBocwRNRBVmGUZKLFEeDAxrKxEOsALGHk=", + "zh:079162e3cb8e3e1e24ae5cf099f09a545b47e5acfd405d1ac2a478cdfc27a0e1", + "zh:263e8e027a655452420ff621362fbcd8b7f765120adec2881d5a3c906555de16", + "zh:2ef4e642f6f106925112726a7c5eb862578406e6aa52e9623ef56d0207ab8121", + "zh:346729435b3be07830973e5b9173dba43add86589fe5a25c13d029d940b935de", + "zh:5ebc3eaaf33ebaaa7b68499a65978f46413c4d377c1b5e2ba143392664fab231", + "zh:75f74966e7ca1ed328b217e95e59a2c54135bcea0f4ec8e4202cef89c2ba4996", + "zh:8041807080b0134517d2d346fdd72d7b67a6673ede6b99bc7f6eb8c148449285", + "zh:904352a3ddbe530934dfa03da54c1a76b96f059a21f77008031e754db51fe0fb", + "zh:92d47ca9ec6c77b688a6e45ff509b287f2333399d2ecb5907d4ec109466857a1", + "zh:a9b7bc9334c069dcafd98037f5317a93d269d67e792f45d5879f9afc7bb65ca7", + "zh:b38ce8512d14a6f4073a41442c3c6b88b91e4419dc030d63633215c2e14cd8c9", + "zh:c8dc7893fcd9b233cc17acc0ee0f37d9873b93a441f6e1af1c1042c67fb81ab8", + "zh:db66259c5bdb8a11e7b3796358d182a2a00ce557ffaeae31f8be784806f6e273", + ] +} diff --git a/terraform/bambi.tf b/terraform/bambi.tf index 2d23c9b..808d285 100644 --- a/terraform/bambi.tf +++ b/terraform/bambi.tf @@ -5,9 +5,14 @@ terraform { version = "2.2.0" } hcloud = { - source = "hetznercloud/hcloud" + source = "hetznercloud/hcloud" version = "1.35.2" } + digitalocean = { + source = "digitalocean/digitalocean" + version = "2.39.2" + } + } required_version = ">= 1.0" } @@ -24,19 +29,19 @@ variable "HETZNERDNS_TOKEN" { } variable "home_location" { - type = string - nullable = false - default = "fsn1" + type = string + nullable = false + default = "fsn1" } variable "hetznerdns_zone" { - type = string - default = null + type = string + default = null } variable "subdomain" { - type = string - default = null + type = string + default = null } locals { diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index 32a6408..1d677ae 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -1,13 +1,13 @@ variable "checker_type" { - type = string - default = "cpx31" - nullable = false + type = string + default = "cpx31" + nullable = false } variable "checker_count" { - type = number - default = 0 - nullable = false + type = number + default = 0 + nullable = false } data "hcloud_image" "bambichecker" { @@ -39,6 +39,8 @@ resource "hcloud_server" "bambichecker" { router_ips = hcloud_floating_ip.bambirouter_ip, elk = hcloud_floating_ip.bambielk_ip.ip_address, engine = hcloud_floating_ip.bambiengine_ip.ip_address, + proxy_count = var.proxy_count, + proxy_url = "proxy${local.subdomain}.${var.hetznerdns_zone}", } ) } diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf index 407f6b8..1b96669 100644 --- a/terraform/bambivulnbox.tf +++ b/terraform/bambivulnbox.tf @@ -1,13 +1,13 @@ variable "vulnbox_type" { - type = string - default = "cpx21" - nullable = false + type = string + default = "cpx21" + nullable = false } variable "vulnbox_count" { - type = number - default = 0 - nullable = false + type = number + default = 0 + nullable = false } data "hcloud_image" "bambivulnbox" { @@ -38,7 +38,9 @@ resource "hcloud_server" "bambivulnbox" { wgconf = file("../config/export/terraform/team${count.index + 1}/game.conf"), index = count.index, id = "${count.index + 1}", - router_ips = hcloud_floating_ip.bambirouter_ip + router_ips = hcloud_floating_ip.bambirouter_ip, + proxy_count = var.proxy_count, + proxy_url = "proxy${local.subdomain}.${var.hetznerdns_zone}", } ) } diff --git a/terraform/proxy.tf b/terraform/proxy.tf new file mode 100644 index 0000000..fc27226 --- /dev/null +++ b/terraform/proxy.tf @@ -0,0 +1,48 @@ +variable "DO_TOKEN" { + type = string +} + +variable "proxy_count" { + type = number + default = 0 +} + +provider "digitalocean" { + token = var.DO_TOKEN +} + +data "digitalocean_ssh_keys" "keys" { + sort { + key = "name" + direction = "asc" + } +} + +# Create a new Web Droplet in the nyc2 region +resource "digitalocean_droplet" "proxy" { + count = var.proxy_count + image = "ubuntu-24-04-x64" + name = "proxy${count.index + 1}" + region = "fra1" + size = "s-1vcpu-512mb-10gb" + ssh_keys = [for k in data.digitalocean_ssh_keys.keys.ssh_keys : k.id] + user_data = templatefile("user_data_proxy.tftpl", {}) +} + +resource "hetznerdns_record" "proxy_dns" { + count = var.proxy_count + zone_id = data.hetznerdns_zone.zone[0].id + name = "proxy${count.index + 1}${local.subdomain}" + value = digitalocean_droplet.proxy[count.index].ipv4_address + type = "A" + ttl = 60 +} + +resource "hetznerdns_record" "proxy_rr_dns" { + count = var.proxy_count + zone_id = data.hetznerdns_zone.zone[0].id + name = "proxy${local.subdomain}" + value = digitalocean_droplet.proxy[count.index].ipv4_address + type = "A" + ttl = 60 +} diff --git a/terraform/squid_acl.txt b/terraform/squid_acl.txt new file mode 100644 index 0000000..d88b7e0 --- /dev/null +++ b/terraform/squid_acl.txt @@ -0,0 +1,79 @@ +acl hetzner src 116.202.0.0/16 +acl hetzner src 116.203.0.0/16 +acl hetzner src 128.140.0.0/17 +acl hetzner src 135.181.0.0/16 +acl hetzner src 136.243.0.0/16 +acl hetzner src 138.201.0.0/16 +acl hetzner src 142.132.128.0/17 +acl hetzner src 144.76.0.0/16 +acl hetzner src 148.251.0.0/16 +acl hetzner src 157.180.0.0/17 +acl hetzner src 157.90.0.0/16 +acl hetzner src 159.69.0.0/16 +acl hetzner src 162.55.0.0/16 +acl hetzner src 167.233.0.0/16 +acl hetzner src 167.235.0.0/16 +acl hetzner src 168.119.0.0/16 +acl hetzner src 171.25.225.0/24 +acl hetzner src 176.9.0.0/16 +acl hetzner src 178.212.75.0/24 +acl hetzner src 178.63.0.0/16 +acl hetzner src 185.107.52.0/22 +acl hetzner src 185.126.28.0/22 +acl hetzner src 185.157.176.0/23 +acl hetzner src 185.157.178.0/23 +acl hetzner src 185.157.83.0/24 +acl hetzner src 185.171.224.0/22 +acl hetzner src 185.189.228.0/24 +acl hetzner src 185.189.229.0/24 +acl hetzner src 185.189.230.0/24 +acl hetzner src 185.189.231.0/24 +acl hetzner src 185.213.45.0/24 +acl hetzner src 185.216.237.0/24 +acl hetzner src 185.226.99.0/24 +acl hetzner src 185.228.8.0/23 +acl hetzner src 185.242.76.0/24 +acl hetzner src 185.253.111.0/24 +acl hetzner src 185.50.120.0/23 +acl hetzner src 188.245.0.0/16 +acl hetzner src 188.34.128.0/17 +acl hetzner src 188.40.0.0/16 +acl hetzner src 193.110.6.0/23 +acl hetzner src 193.163.198.0/24 +acl hetzner src 193.25.170.0/23 +acl hetzner src 194.42.180.0/22 +acl hetzner src 194.42.184.0/22 +acl hetzner src 194.62.106.0/24 +acl hetzner src 195.201.0.0/16 +acl hetzner src 195.248.224.0/24 +acl hetzner src 195.60.226.0/24 +acl hetzner src 197.242.84.0/22 +acl hetzner src 201.131.3.0/24 +acl hetzner src 204.29.146.0/24 +acl hetzner src 213.133.96.0/19 +acl hetzner src 213.232.193.0/24 +acl hetzner src 213.239.192.0/18 +acl hetzner src 216.55.108.0/22 +acl hetzner src 23.88.0.0/17 +acl hetzner src 37.27.0.0/16 +acl hetzner src 45.145.227.0/24 +acl hetzner src 46.4.0.0/16 +acl hetzner src 49.12.0.0/16 +acl hetzner src 49.13.0.0/16 +acl hetzner src 5.75.128.0/17 +acl hetzner src 5.9.0.0/16 +acl hetzner src 65.108.0.0/16 +acl hetzner src 65.109.0.0/16 +acl hetzner src 65.21.0.0/16 +acl hetzner src 78.138.62.0/24 +acl hetzner src 78.46.0.0/15 +acl hetzner src 85.10.192.0/18 +acl hetzner src 88.198.0.0/16 +acl hetzner src 88.99.0.0/16 +acl hetzner src 89.42.83.0/24 +acl hetzner src 91.107.128.0/17 +acl hetzner src 91.190.240.0/21 +acl hetzner src 91.233.8.0/22 +acl hetzner src 94.130.0.0/16 +acl hetzner src 95.216.0.0/16 +acl hetzner src 95.217.0.0/16 diff --git a/terraform/user_data_checker.tftpl b/terraform/user_data_checker.tftpl index 539f733..7707aa9 100644 --- a/terraform/user_data_checker.tftpl +++ b/terraform/user_data_checker.tftpl @@ -1,6 +1,20 @@ #!/bin/bash set -e +%{ if proxy_count > 0 ~} +cat <<EOF > ~/.docker/config.json +{ + "proxies": { + "default": { + "httpProxy": "${proxy_url}", + "httpsProxy": "${proxy_url}", + "noProxy": "localhost,127.0.0.0/8" + } + } +} +EOF +%{ endif } + # Network mv "/etc/wireguard/checker${id}.conf" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" diff --git a/terraform/user_data_proxy.tftpl b/terraform/user_data_proxy.tftpl new file mode 100644 index 0000000..2ff47e0 --- /dev/null +++ b/terraform/user_data_proxy.tftpl @@ -0,0 +1,14 @@ +#!/bin/bash + +apt install -y squid + +cat << EOF > /etc/squid/conf.d/hetzner_whitelist.conf +${file("squid_acl.txt")} + +http_access allow hetzner +EOF + +curl https://github.com/ldruschk.keys >> /root/.ssh/authorized_keys +curl https://github.com/trolldemorted.keys >> /root/.ssh/authorized_keys + +systemctl restart squid diff --git a/terraform/user_data_vulnbox.tftpl b/terraform/user_data_vulnbox.tftpl index 2fa1047..448bc4e 100644 --- a/terraform/user_data_vulnbox.tftpl +++ b/terraform/user_data_vulnbox.tftpl @@ -1,6 +1,20 @@ #!/bin/bash set -e +%{ if proxy_count > 0 ~} +cat <<EOF > ~/.docker/config.json +{ + "proxies": { + "default": { + "httpProxy": "${proxy_url}", + "httpsProxy": "${proxy_url}", + "noProxy": "localhost,127.0.0.0/8" + } + } +} +EOF +%{ endif } + # Network cat > /etc/wireguard/game.conf <<EOF ${wgconf} From c7cc7738c4bb79055e43b96d54ec41cd78dd73fa Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 19 Jul 2024 16:07:31 +0000 Subject: [PATCH 56/61] =?UTF-8?q?w=C3=B6rkw=C3=B6rk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ansible/roles/vuln_services/tasks/main.yml | 19 +++++++ .../files/docker-compose.yml | 4 ++ .../roles/vulnbox_enomoloch/tasks/main.yml | 2 +- docker-compose.yml | 2 + packer/bambichecker.json | 2 +- packer/bambivulnbox.json | 2 +- proxy/.terraform.lock.hcl | 42 ++++++++++++++++ {terraform => proxy}/proxy.tf | 50 ++++++++++++++++++- {terraform => proxy}/squid_acl.txt | 0 proxy/terraform.tfvars.sample | 2 + {terraform => proxy}/user_data_proxy.tftpl | 3 +- terraform/.terraform.lock.hcl | 45 +++++++++-------- terraform/bambi.tf | 1 - terraform/bambichecker.tf | 4 +- terraform/bambielk.tf | 4 +- terraform/bambiengine.tf | 4 +- terraform/bambirouter.tf | 4 +- terraform/bambivulnbox.tf | 6 +-- terraform/user_data_checker.tftpl | 14 ------ terraform/user_data_vulnbox.tftpl | 14 ------ 20 files changed, 155 insertions(+), 69 deletions(-) create mode 100644 proxy/.terraform.lock.hcl rename {terraform => proxy}/proxy.tf (58%) rename {terraform => proxy}/squid_acl.txt (100%) create mode 100644 proxy/terraform.tfvars.sample rename {terraform => proxy}/user_data_proxy.tftpl (65%) diff --git a/ansible/roles/vuln_services/tasks/main.yml b/ansible/roles/vuln_services/tasks/main.yml index ff505ba..837852c 100644 --- a/ansible/roles/vuln_services/tasks/main.yml +++ b/ansible/roles/vuln_services/tasks/main.yml @@ -5,6 +5,25 @@ state: directory mode: "0755" +- name: Configure docker http proxy + shell: | + echo '{ + "proxies": { + "http-proxy": "http://proxy.prod.bambi.ovh:3128", + "https-proxy": "http://proxy.prod.bambi.ovh:3128" + }, + "log-driver": "json-file", + "log-opts": { + "max-size": "300m", + "max-file": "3" + }, + "default-address-pools": [{ + "base": "172.17.0.0/16", + "size": 24 + }] + }' > /etc/docker/daemon.json + systemctl restart docker + - name: Clean local services cache become: false file: diff --git a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml index 7e5d3da..af0de97 100644 --- a/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml +++ b/ansible/roles/vulnbox_enomoloch/files/docker-compose.yml @@ -1,12 +1,16 @@ version: '3' services: moloch: + restart: unless-stopped image: ghcr.io/enoflag/enoarkime:5.3.0 ports: - 8005:8005 volumes: - "/pcaps:/opt/arkime/raw" elasticsearch: + restart: unless-stopped image: elasticsearch:7.14.2 environment: + - xpack.security.enabled=false - discovery.type=single-node + - ingest.geoip.downloader.enabled=false diff --git a/ansible/roles/vulnbox_enomoloch/tasks/main.yml b/ansible/roles/vulnbox_enomoloch/tasks/main.yml index 88e4783..e5e9281 100644 --- a/ansible/roles/vulnbox_enomoloch/tasks/main.yml +++ b/ansible/roles/vulnbox_enomoloch/tasks/main.yml @@ -50,7 +50,7 @@ state: directory owner: tcpdump group: tcpdump - mode: "0644" + mode: "0755" # 0644 is not enough - name: Enable and start enodump service: diff --git a/docker-compose.yml b/docker-compose.yml index 7d2be45..fa3414c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,5 +5,7 @@ services: - HCLOUD_TOKEN=${HCLOUD_TOKEN} # for packer - TF_VAR_HCLOUD_TOKEN=${HCLOUD_TOKEN} # for terraform - TF_VAR_HETZNERDNS_TOKEN=${HETZNERDNS_TOKEN} # for terraform + - TF_VAR_DO_TOKEN=${DO_TOKEN} # for terraform (optional) + - DO_TOKEN=${DO_TOKEN} # for packer/ansible (optional) volumes: - ".:/bambictf" diff --git a/packer/bambichecker.json b/packer/bambichecker.json index 35afac5..12dd9e0 100644 --- a/packer/bambichecker.json +++ b/packer/bambichecker.json @@ -15,7 +15,7 @@ "type": "hcloud", "image": "ubuntu-22.04", "location": "fsn1", - "server_type": "cx11", + "server_type": "cx22", "ssh_username": "root", "snapshot_name": "bambichecker-{{timestamp}}", "snapshot_labels": { diff --git a/packer/bambivulnbox.json b/packer/bambivulnbox.json index 85f6e23..ee2aa46 100644 --- a/packer/bambivulnbox.json +++ b/packer/bambivulnbox.json @@ -15,7 +15,7 @@ "type": "hcloud", "image": "ubuntu-22.04", "location": "fsn1", - "server_type": "cx11", + "server_type": "cx32", "ssh_username": "root", "snapshot_name": "bambivulnbox-{{timestamp}}", "snapshot_labels": { diff --git a/proxy/.terraform.lock.hcl b/proxy/.terraform.lock.hcl new file mode 100644 index 0000000..c43767b --- /dev/null +++ b/proxy/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/digitalocean/digitalocean" { + version = "2.39.2" + constraints = "2.39.2" + hashes = [ + "h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=", + "zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f", + "zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0", + "zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f", + "zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee", + "zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea", + "zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5", + "zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e", + "zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb", + "zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761", + "zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9", + "zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf", + "zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720", + "zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad", + "zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0", + "zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f", + "zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba", + ] +} + +provider "registry.terraform.io/timohirt/hetznerdns" { + version = "2.2.0" + constraints = "2.2.0" + hashes = [ + "h1:HyskQAglrOueur79gSCBgx9MNDOs0tz39aNYQiFgxz8=", + "zh:5bb0ab9f62be3ed92070235e507f3c290491d51391ef4edcc70df53b65a83019", + "zh:5ccdfac7284f5515ac3cff748336b77f21c64760e429e811a1eeefa8ebb86e12", + "zh:687c35665139ae37c291e99085be2e38071f6b355c4e1e8957c5a6a3bcdf9caf", + "zh:6de27f0d0d1513b3a4b7e81923b4a8506c52759bd466e2b4f8156997b0478931", + "zh:85770a9199a4c2d16ca41538d7a0f7a7bfc060678104a1faac19213e6f0a800c", + "zh:a5ff723774a9ccfb27d5766c5e6713537f74dd94496048c89c5d64dba597e59e", + "zh:bf9ab76fd37cb8aebb6868d73cbe8c08cee36fc25224cc1ef5949efa3c34b06c", + "zh:db998fe3bdcd4902e99fa470bb3f355883170cf4c711c8da0b5f1f4510f1be41", + ] +} diff --git a/terraform/proxy.tf b/proxy/proxy.tf similarity index 58% rename from terraform/proxy.tf rename to proxy/proxy.tf index fc27226..9897f10 100644 --- a/terraform/proxy.tf +++ b/proxy/proxy.tf @@ -1,16 +1,62 @@ +terraform { + required_providers { + hetznerdns = { + source = "timohirt/hetznerdns" + version = "2.2.0" + } + digitalocean = { + source = "digitalocean/digitalocean" + version = "2.39.2" + } + + } + required_version = ">= 1.0" +} + variable "DO_TOKEN" { - type = string + type = string + nullable = false + sensitive = true +} + +variable "HETZNERDNS_TOKEN" { + type = string + nullable = false + sensitive = true } variable "proxy_count" { type = number - default = 0 + default = 1 +} + +variable "hetznerdns_zone" { + type = string + default = null +} + +variable "subdomain" { + type = string + default = null +} + +locals { + subdomain = var.subdomain != null ? ".${var.subdomain}" : "" } provider "digitalocean" { token = var.DO_TOKEN } +provider "hetznerdns" { + apitoken = var.HETZNERDNS_TOKEN +} + +data "hetznerdns_zone" "zone" { + count = var.hetznerdns_zone != null ? 1 : 0 + name = var.hetznerdns_zone +} + data "digitalocean_ssh_keys" "keys" { sort { key = "name" diff --git a/terraform/squid_acl.txt b/proxy/squid_acl.txt similarity index 100% rename from terraform/squid_acl.txt rename to proxy/squid_acl.txt diff --git a/proxy/terraform.tfvars.sample b/proxy/terraform.tfvars.sample new file mode 100644 index 0000000..7a04848 --- /dev/null +++ b/proxy/terraform.tfvars.sample @@ -0,0 +1,2 @@ +hetznerdns_zone = "bambi.ovh" +subdomain = "prod" diff --git a/terraform/user_data_proxy.tftpl b/proxy/user_data_proxy.tftpl similarity index 65% rename from terraform/user_data_proxy.tftpl rename to proxy/user_data_proxy.tftpl index 2ff47e0..9b2faf1 100644 --- a/terraform/user_data_proxy.tftpl +++ b/proxy/user_data_proxy.tftpl @@ -9,6 +9,7 @@ http_access allow hetzner EOF curl https://github.com/ldruschk.keys >> /root/.ssh/authorized_keys -curl https://github.com/trolldemorted.keys >> /root/.ssh/authorized_keys +curl https://github.com/Trolldemorted.keys >> /root/.ssh/authorized_keys +curl https://github.com/MMunier.keys >> /root/.ssh/authorized_keys systemctl restart squid diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 03461e5..efaf011 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,6 +1,30 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/digitalocean/digitalocean" { + version = "2.39.2" + constraints = "2.39.2" + hashes = [ + "h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=", + "zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f", + "zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0", + "zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f", + "zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee", + "zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea", + "zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5", + "zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e", + "zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb", + "zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761", + "zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9", + "zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf", + "zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720", + "zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad", + "zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0", + "zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f", + "zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba", + ] +} + provider "registry.terraform.io/hetznercloud/hcloud" { version = "1.35.2" constraints = "1.35.2" @@ -38,24 +62,3 @@ provider "registry.terraform.io/timohirt/hetznerdns" { "zh:db998fe3bdcd4902e99fa470bb3f355883170cf4c711c8da0b5f1f4510f1be41", ] } - -provider "registry.terraform.io/vancluever/acme" { - version = "2.24.2" - constraints = "2.24.2" - hashes = [ - "h1:zxpXpKMgBbhBocwRNRBVmGUZKLFEeDAxrKxEOsALGHk=", - "zh:079162e3cb8e3e1e24ae5cf099f09a545b47e5acfd405d1ac2a478cdfc27a0e1", - "zh:263e8e027a655452420ff621362fbcd8b7f765120adec2881d5a3c906555de16", - "zh:2ef4e642f6f106925112726a7c5eb862578406e6aa52e9623ef56d0207ab8121", - "zh:346729435b3be07830973e5b9173dba43add86589fe5a25c13d029d940b935de", - "zh:5ebc3eaaf33ebaaa7b68499a65978f46413c4d377c1b5e2ba143392664fab231", - "zh:75f74966e7ca1ed328b217e95e59a2c54135bcea0f4ec8e4202cef89c2ba4996", - "zh:8041807080b0134517d2d346fdd72d7b67a6673ede6b99bc7f6eb8c148449285", - "zh:904352a3ddbe530934dfa03da54c1a76b96f059a21f77008031e754db51fe0fb", - "zh:92d47ca9ec6c77b688a6e45ff509b287f2333399d2ecb5907d4ec109466857a1", - "zh:a9b7bc9334c069dcafd98037f5317a93d269d67e792f45d5879f9afc7bb65ca7", - "zh:b38ce8512d14a6f4073a41442c3c6b88b91e4419dc030d63633215c2e14cd8c9", - "zh:c8dc7893fcd9b233cc17acc0ee0f37d9873b93a441f6e1af1c1042c67fb81ab8", - "zh:db66259c5bdb8a11e7b3796358d182a2a00ce557ffaeae31f8be784806f6e273", - ] -} diff --git a/terraform/bambi.tf b/terraform/bambi.tf index 808d285..5d20619 100644 --- a/terraform/bambi.tf +++ b/terraform/bambi.tf @@ -12,7 +12,6 @@ terraform { source = "digitalocean/digitalocean" version = "2.39.2" } - } required_version = ">= 1.0" } diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index 1d677ae..5d6b451 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -27,7 +27,7 @@ resource "hetznerdns_record" "bambchecker_dns" { resource "hcloud_server" "bambichecker" { count = var.checker_count - name = "checker${count.index + 1}" + name = "checker${count.index + 1}${local.subdomain}" image = data.hcloud_image.bambichecker.id location = var.home_location server_type = var.checker_type @@ -39,8 +39,6 @@ resource "hcloud_server" "bambichecker" { router_ips = hcloud_floating_ip.bambirouter_ip, elk = hcloud_floating_ip.bambielk_ip.ip_address, engine = hcloud_floating_ip.bambiengine_ip.ip_address, - proxy_count = var.proxy_count, - proxy_url = "proxy${local.subdomain}.${var.hetznerdns_zone}", } ) } diff --git a/terraform/bambielk.tf b/terraform/bambielk.tf index 13a5ee7..66ab790 100644 --- a/terraform/bambielk.tf +++ b/terraform/bambielk.tf @@ -23,7 +23,7 @@ data "hcloud_image" "bambielk" { resource "hcloud_floating_ip" "bambielk_ip" { type = "ipv4" - name = "elk" + name = "elk${local.subdomain}" home_location = var.home_location } @@ -44,7 +44,7 @@ resource "hetznerdns_record" "bambielk_dns" { resource "hcloud_server" "bambielk" { count = var.elk_count - name = "bambielk" + name = "elk${local.subdomain}" image = data.hcloud_image.bambielk[0].id location = var.home_location server_type = var.elk_type diff --git a/terraform/bambiengine.tf b/terraform/bambiengine.tf index 8aae16c..98e1de1 100644 --- a/terraform/bambiengine.tf +++ b/terraform/bambiengine.tf @@ -23,7 +23,7 @@ data "hcloud_image" "bambiengine" { resource "hcloud_floating_ip" "bambiengine_ip" { type = "ipv4" - name = "engine" + name = "engine${local.subdomain}" home_location = var.home_location } @@ -44,7 +44,7 @@ resource "hetznerdns_record" "bambiengine_dns" { resource "hcloud_server" "bambiengine" { count = var.engine_count - name = "engine" + name = "engine${local.subdomain}" image = data.hcloud_image.bambiengine[0].id location = var.home_location server_type = var.engine_type diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf index f4e7d5b..27a1d1f 100644 --- a/terraform/bambirouter.tf +++ b/terraform/bambirouter.tf @@ -23,7 +23,7 @@ data "hcloud_image" "bambirouter" { resource "hcloud_floating_ip" "bambirouter_ip" { count = var.router_count type = "ipv4" - name = "router${count.index + 1}" + name = "router${count.index + 1}${local.subdomain}" home_location = var.home_location } @@ -44,7 +44,7 @@ resource "hetznerdns_record" "bambirouter_dns" { resource "hcloud_server" "bambirouter" { count = var.router_count - name = "router${count.index + 1}" + name = "router${count.index + 1}${local.subdomain}" image = data.hcloud_image.bambirouter.id location = var.home_location server_type = var.router_type diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf index 1b96669..774c515 100644 --- a/terraform/bambivulnbox.tf +++ b/terraform/bambivulnbox.tf @@ -12,7 +12,7 @@ variable "vulnbox_count" { data "hcloud_image" "bambivulnbox" { with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null - name = var.vulnbox_count > 0 ? null : "debian-10" + name = var.vulnbox_count > 0 ? null : "debian-12" most_recent = true } @@ -26,7 +26,7 @@ resource "hetznerdns_record" "bambivulnbox_dns" { } resource "hcloud_server" "bambivulnbox" { - name = "vulnbox${count.index + 1}" + name = "vulnbox${count.index + 1}${local.subdomain}" image = data.hcloud_image.bambivulnbox.id location = var.home_location server_type = var.vulnbox_type @@ -39,8 +39,6 @@ resource "hcloud_server" "bambivulnbox" { index = count.index, id = "${count.index + 1}", router_ips = hcloud_floating_ip.bambirouter_ip, - proxy_count = var.proxy_count, - proxy_url = "proxy${local.subdomain}.${var.hetznerdns_zone}", } ) } diff --git a/terraform/user_data_checker.tftpl b/terraform/user_data_checker.tftpl index 7707aa9..539f733 100644 --- a/terraform/user_data_checker.tftpl +++ b/terraform/user_data_checker.tftpl @@ -1,20 +1,6 @@ #!/bin/bash set -e -%{ if proxy_count > 0 ~} -cat <<EOF > ~/.docker/config.json -{ - "proxies": { - "default": { - "httpProxy": "${proxy_url}", - "httpsProxy": "${proxy_url}", - "noProxy": "localhost,127.0.0.0/8" - } - } -} -EOF -%{ endif } - # Network mv "/etc/wireguard/checker${id}.conf" "/etc/wireguard/internal.conf" sed -i -e "s#\[\[ENGINE_ADDRESS\]\]#${engine}#g" "/etc/wireguard/internal.conf" diff --git a/terraform/user_data_vulnbox.tftpl b/terraform/user_data_vulnbox.tftpl index 448bc4e..2fa1047 100644 --- a/terraform/user_data_vulnbox.tftpl +++ b/terraform/user_data_vulnbox.tftpl @@ -1,20 +1,6 @@ #!/bin/bash set -e -%{ if proxy_count > 0 ~} -cat <<EOF > ~/.docker/config.json -{ - "proxies": { - "default": { - "httpProxy": "${proxy_url}", - "httpsProxy": "${proxy_url}", - "noProxy": "localhost,127.0.0.0/8" - } - } -} -EOF -%{ endif } - # Network cat > /etc/wireguard/game.conf <<EOF ${wgconf} From f643f2b4cee72e6e46535b85de013ada79b31aa8 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 19 Jul 2024 16:13:41 +0000 Subject: [PATCH 57/61] allow pinging flag submission endpoint from vpn connections --- ansible/roles/router_iptables/tasks/main.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ansible/roles/router_iptables/tasks/main.yml b/ansible/roles/router_iptables/tasks/main.yml index 3d57c8f..41aacfc 100644 --- a/ansible/roles/router_iptables/tasks/main.yml +++ b/ansible/roles/router_iptables/tasks/main.yml @@ -81,7 +81,7 @@ jump: DNAT to_destination: 192.168.1.0 -- name: Allow pinging the flag submission endpoint +- name: Allow pinging the flag submission endpoint from vulnboxes iptables: chain: FORWARD in_interface: router @@ -91,6 +91,16 @@ icmp_type: "8" jump: ACCEPT +- name: Allow pinging the flag submission endpoint from vpn connections + iptables: + chain: FORWARD + in_interface: team+ + out_interface: internal + destination: 192.168.1.0 + protocol: icmp + icmp_type: "8" + jump: ACCEPT + - name: Allow flag submission traffic from vulnboxes iptables: chain: FORWARD From bb4d25caefa7aa6e1ff148f2d6e60871696a05a2 Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 19 Jul 2024 16:28:12 +0000 Subject: [PATCH 58/61] some fixes --- terraform/bambichecker.tf | 4 ++-- terraform/bambielk.tf | 4 ++-- terraform/bambiengine.tf | 4 ++-- terraform/bambirouter.tf | 4 ++-- terraform/bambivulnbox.tf | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform/bambichecker.tf b/terraform/bambichecker.tf index 32a6408..bf24890 100644 --- a/terraform/bambichecker.tf +++ b/terraform/bambichecker.tf @@ -12,7 +12,7 @@ variable "checker_count" { data "hcloud_image" "bambichecker" { with_selector = var.checker_count > 0 ? "type=bambichecker" : null - name = var.checker_count > 0 ? null : "debian-10" + name = var.checker_count > 0 ? null : "debian-12" most_recent = true } @@ -27,7 +27,7 @@ resource "hetznerdns_record" "bambchecker_dns" { resource "hcloud_server" "bambichecker" { count = var.checker_count - name = "checker${count.index + 1}" + name = "checker${count.index + 1}${local.subdomain}" image = data.hcloud_image.bambichecker.id location = var.home_location server_type = var.checker_type diff --git a/terraform/bambielk.tf b/terraform/bambielk.tf index 13a5ee7..8c8a978 100644 --- a/terraform/bambielk.tf +++ b/terraform/bambielk.tf @@ -17,7 +17,7 @@ variable "elk_count" { data "hcloud_image" "bambielk" { count = var.elk_count > 0 ? 1 : 0 with_selector = var.elk_count > 0 ? "type=bambielk" : null - name = var.elk_count > 0 ? null : "debian-10" + name = var.elk_count > 0 ? null : "debian-12" most_recent = true } @@ -44,7 +44,7 @@ resource "hetznerdns_record" "bambielk_dns" { resource "hcloud_server" "bambielk" { count = var.elk_count - name = "bambielk" + name = "elk${local.subdomain}" image = data.hcloud_image.bambielk[0].id location = var.home_location server_type = var.elk_type diff --git a/terraform/bambiengine.tf b/terraform/bambiengine.tf index 8aae16c..5f00626 100644 --- a/terraform/bambiengine.tf +++ b/terraform/bambiengine.tf @@ -17,7 +17,7 @@ variable "engine_count" { data "hcloud_image" "bambiengine" { count = var.engine_count > 0 ? 1 : 0 with_selector = var.engine_count > 0 ? "type=bambiengine" : null - name = var.engine_count > 0 ? null : "debian-10" + name = var.engine_count > 0 ? null : "debian-12" most_recent = true } @@ -44,7 +44,7 @@ resource "hetznerdns_record" "bambiengine_dns" { resource "hcloud_server" "bambiengine" { count = var.engine_count - name = "engine" + name = "engine${local.subdomain}" image = data.hcloud_image.bambiengine[0].id location = var.home_location server_type = var.engine_type diff --git a/terraform/bambirouter.tf b/terraform/bambirouter.tf index f4e7d5b..f8236f4 100644 --- a/terraform/bambirouter.tf +++ b/terraform/bambirouter.tf @@ -16,7 +16,7 @@ locals { data "hcloud_image" "bambirouter" { with_selector = var.router_count > 0 ? "type=bambirouter" : null - name = var.router_count > 0 ? null : "debian-10" + name = var.router_count > 0 ? null : "debian-12" most_recent = true } @@ -44,7 +44,7 @@ resource "hetznerdns_record" "bambirouter_dns" { resource "hcloud_server" "bambirouter" { count = var.router_count - name = "router${count.index + 1}" + name = "router${count.index + 1}${local.subdomain}" image = data.hcloud_image.bambirouter.id location = var.home_location server_type = var.router_type diff --git a/terraform/bambivulnbox.tf b/terraform/bambivulnbox.tf index 407f6b8..5d01755 100644 --- a/terraform/bambivulnbox.tf +++ b/terraform/bambivulnbox.tf @@ -12,7 +12,7 @@ variable "vulnbox_count" { data "hcloud_image" "bambivulnbox" { with_selector = var.vulnbox_count > 0 ? "type=bambivulnbox" : null - name = var.vulnbox_count > 0 ? null : "debian-10" + name = var.vulnbox_count > 0 ? null : "debian-12" most_recent = true } @@ -26,7 +26,7 @@ resource "hetznerdns_record" "bambivulnbox_dns" { } resource "hcloud_server" "bambivulnbox" { - name = "vulnbox${count.index + 1}" + name = "vulnbox${count.index + 1}${local.subdomain}" image = data.hcloud_image.bambivulnbox.id location = var.home_location server_type = var.vulnbox_type From 45f14a0940b7daad48537d4fb102242d60ca8b5f Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 19 Jul 2024 16:31:58 +0000 Subject: [PATCH 59/61] remove do provider in terraform --- terraform/.terraform.lock.hcl | 24 ------------------------ terraform/bambi.tf | 4 ---- 2 files changed, 28 deletions(-) diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index efaf011..8d9c4ea 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,30 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/digitalocean/digitalocean" { - version = "2.39.2" - constraints = "2.39.2" - hashes = [ - "h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=", - "zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f", - "zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0", - "zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f", - "zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee", - "zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea", - "zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5", - "zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e", - "zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb", - "zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761", - "zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9", - "zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf", - "zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720", - "zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad", - "zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0", - "zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f", - "zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba", - ] -} - provider "registry.terraform.io/hetznercloud/hcloud" { version = "1.35.2" constraints = "1.35.2" diff --git a/terraform/bambi.tf b/terraform/bambi.tf index 5d20619..25c2abd 100644 --- a/terraform/bambi.tf +++ b/terraform/bambi.tf @@ -8,10 +8,6 @@ terraform { source = "hetznercloud/hcloud" version = "1.35.2" } - digitalocean = { - source = "digitalocean/digitalocean" - version = "2.39.2" - } } required_version = ">= 1.0" } From 6ea20a99eb1b9bb4b3d767d8788601a62b9855fe Mon Sep 17 00:00:00 2001 From: Benedikt Radtke <benediktradtke@gmail.com> Date: Fri, 19 Jul 2024 19:05:02 +0000 Subject: [PATCH 60/61] wip --- ansible/roles/bambixploit/files/bambixploit.json | 4 ++-- ansible/roles/vulnbox_enomoloch/tasks/main.yml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ansible/roles/bambixploit/files/bambixploit.json b/ansible/roles/bambixploit/files/bambixploit.json index 3fb6d16..cbb3ceb 100644 --- a/ansible/roles/bambixploit/files/bambixploit.json +++ b/ansible/roles/bambixploit/files/bambixploit.json @@ -1,7 +1,7 @@ { "flagRegex": "ENO[A-Za-z0-9+\/=]{48}", - "targetsUrl": "http://10.0.13.37:5001/api/data/ips", + "targetsUrl": "https://8.enowars.com/api/data/ips", "submissionAddress": "10.0.13.37", "submissionPort": 1337, "interval": 50 -} \ No newline at end of file +} diff --git a/ansible/roles/vulnbox_enomoloch/tasks/main.yml b/ansible/roles/vulnbox_enomoloch/tasks/main.yml index e5e9281..5bb2db9 100644 --- a/ansible/roles/vulnbox_enomoloch/tasks/main.yml +++ b/ansible/roles/vulnbox_enomoloch/tasks/main.yml @@ -39,6 +39,11 @@ mode: "0644" changed_when: false +- name: Pull EnoMoloch images + shell: docker compose pull # noqa: command-instead-of-shell + args: + chdir: /services/EnoMoloch + - name: Reload system daemon systemd: name: "enodump" # ansible <2.4 always requires 'name' From fa5c857dec80142f4078240d458e0718a40b2fc7 Mon Sep 17 00:00:00 2001 From: Lucas Druschke <ldruschk@posteo.de> Date: Sun, 21 Jul 2024 15:03:07 +0200 Subject: [PATCH 61/61] add docker-publish.yaml to publish latest docker images --- .github/workflows/docker-publish.yaml | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/docker-publish.yaml diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml new file mode 100644 index 0000000..3f1d50e --- /dev/null +++ b/.github/workflows/docker-publish.yaml @@ -0,0 +1,33 @@ +name: Docker Publish Latest + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - main + - master + +env: + IMAGE_NAME: bambictf + +jobs: + # Push image to GitHub Packages. + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Log into GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:latest + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker image build -t $IMAGE_ID . + docker image push $IMAGE_ID