Merge branch 'main' into oonimeasurements

ansible/README.md

@@ -8,7 +8,12 @@ pyenv activate ooni-devops
 
 Install deps:
 ```
-pip install ansible dnspython boto3
+pip install ansible dnspython boto3 passlib
+```
+
+Install ansible galaxy modules:
+```
+ansible-galaxy install -r requirements.yml
 ```
 
 Setup AWS credentials, you should add 2 profiles called `oonidevops_user_dev` and `oonidevops_user_prod` which have access to the development and production environment respectively

ansible/group_vars/all/vars.yml

@@ -26,4 +26,4 @@ ssh_users:
 admin_usernames: [ art, majakomel, mehul, norbel ]
 root_usernames: [ art, mehul ]
 non_admin_usernames: [ agrabeli ]
-deactivated_usernames: [ sbs, federico, sarath ]
+deactivated_usernames: [ sbs, federico, sarath ]

ansible/group_vars/dev/vars.yml

@@ -0,0 +1 @@
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"

ansible/group_vars/prod/vars.yml

@@ -0,0 +1 @@
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"

ansible/host_vars/data.ooni.org

@@ -47,7 +47,14 @@ ssh_users:
  [
  "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKqG1VepfzDdSp3zG27jZq3S9/62CKPLh93F///ht9rf",
  ]
+ vasilis:
+ login: vasilis
+ comment: "Vasilis Ververis"
+ keys:
+ [
+ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMJYsbeTjdma5cKyZISOFQfHbwwlZbWugPx9haeOx1UR"
+ ]
 admin_usernames: [ art, majakomel, mehul, norbel ]
-non_admin_usernames: [ ain, siti, ingrid, joss ]
+non_admin_usernames: [ ain, siti, ingrid, joss, vasilis ]
 jupyterhub_allowed_users: "{{ ssh_users }}"
-admin_group_name: adm
+admin_group_name: adm

ansible/host_vars/oonidata.ooni.org

@@ -0,0 +1,2 @@
+admin_group_name: adm
+tls_cert_dir: /var/lib/dehydrated/certs

ansible/inventory

@@ -1,6 +1,13 @@
 [all]
-monitoring.ooni.org
-openvpn-server1.ooni.io
 # This requires manual setup of ~/.ssh/config
 #codesign-box
+
+[prod]
 data.ooni.org
+oonidata.ooni.org
+monitoring.ooni.org
+openvpn-server1.ooni.io
+notebook.ooni.org
+
+[dev]
+oonidatatest.ooni.nu

ansible/playbook-bootstrap.yml

@@ -0,0 +1,8 @@
+# This playbook is to be run on hosts that don't support bootstrapping the base
+# OS setup with something other than ansible (eg. cloud-init)
+- name: Bootstrap the ssh_users on target host
+ hosts: all
+ remote_user: root
+ roles:
+ - ssh_users
+ - bootstrap

ansible/playbook.yml

@@ -1,4 +1,10 @@
 ---
+- name: Ensure all hosts are bootstrapped correctly
+ hosts: all
+ become: yes
+ roles:
+ - bootstrap
+
 - name: ClickHouse servers
  hosts: clickhouse_servers
  user: admin
@@ -36,6 +42,12 @@
  roles:
  - ssh_users
 
+- name: Deploy oonidata hosts
+ hosts: oonidata.ooni.org
+ become: true
+ roles:
+ - oonidata
+
 # commented out due to the fact it requires manual config of ~/.ssh/config
 #- name: Setup codesign box
 # hosts: codesign-box

ansible/requirements.yml

@@ -1 +1,4 @@
 - src: willshersystems.sshd
+- src: nginxinc.nginx
+- src: geerlingguy.certbot
+- src: geerlingguy.node_exporter

ansible/roles/bootstrap/tasks/main.yml

@@ -0,0 +1,45 @@
+- ansible.builtin.include_role:
+ name: ssh_users
+ tags:
+ - ssh_users
+
+- name: Set the hostname to inventory_hostname
+ ansible.builtin.hostname:
+ name: "{{ inventory_hostname }}"
+
+- name: Install common packages
+ ansible.builtin.apt:
+ name:
+ - bash-completion
+ - ca-certificates
+ - curl
+ - file
+ - git
+ - htop
+ - iotop
+ - lsof
+ - lvm2
+ - man-db
+ - mtr
+ - net-tools
+ - openssl
+ - python3-passlib
+ - rsync
+ - screen
+ - strace
+ - tcpdump
+ - tmux
+ - vim
+ state: latest
+ update_cache: yes
+ install_recommends: no
+
+- ansible.builtin.include_role:
+ name: nftables
+ tags:
+ - nftables
+
+- ansible.builtin.include_role:
+ name: prometheus_node_exporter
+ tags:
+ - node_exporter

ansible/roles/miniconda/defaults/main.yml

@@ -0,0 +1,2 @@
+miniconda_install_dir: /opt/miniconda
+admin_group_name: admin

ansible/roles/miniconda/tasks/install.yml

@@ -0,0 +1,23 @@
+---
+- name: Ensure miniconda directory exists
+ ansible.builtin.file:
+ path: "{{ miniconda_install_dir }}"
+ state: directory
+ owner: miniconda
+ group: "{{ admin_group_name }}"
+
+- name: Download the miniconda installer
+ ansible.builtin.get_url:
+ url: "https://repo.anaconda.com/miniconda/Miniconda3-py312_24.7.1-0-Linux-x86_64.sh"
+ dest: "{{ miniconda_install_dir }}/miniconda.sh"
+ checksum: "sha256:33442cd3813df33dcbb4a932b938ee95398be98344dff4c30f7e757cd2110e4f"
+ mode: "0700"
+
+- name: Run the miniconda installer
+ ansible.builtin.shell: |
+ bash {{ miniconda_install_dir }}/miniconda.sh -b -u -p {{ miniconda_install_dir }}
+
+- name: Delete installer
+ ansible.builtin.file:
+ path: "{{ miniconda_install_dir }}/miniconda.sh"
+ state: absent

ansible/roles/miniconda/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Ensure miniconda user exists
+ ansible.builtin.user:
+ name: miniconda
+ shell: /bin/false
+
+- name: Check if Miniconda is installed
+ ansible.builtin.stat:
+ path: "{{ miniconda_install_dir }}/bin/conda"
+ register: miniconda_bin
+
+- include_tasks: install.yml
+ when: not miniconda_bin.stat.exists
+
+- name: "install conda packages"
+ ansible.builtin.shell:
+ cmd: "{{ miniconda_install_dir }}/bin/conda install -y {{ item }}"
+ loop:
+ - pandas
+ - numpy
+ - altair

ansible/roles/nftables/README.adoc

@@ -0,0 +1,25 @@
+Install nftables based firewall
+
+Set up /etc/ooni/nftables/
+
+Rules for specific services are *not* configured by this role
+
+When creating rules to accept TCP traffic from any IPv4/6 address,
+files are named with the port number to detect collisions.
+
+Example (also see roles/nftables/tasks/main.yml):
+
+/etc/ooni/nftables/tcp/8080.nft
+
+```
+add rule inet filter input tcp dport 8080 counter accept comment "MyService"
+```
+
+
+Otherwise:
+
+/etc/ooni/nftables/tcp/5432_postgres_internal.nft
+
+```
+add rule inet filter input ip saddr { 10.0.0.0/8, 192.168.0.0/16 } tcp dport 5432 counter accept comment "Internal PostgreSQL"
+```

ansible/roles/nftables/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: Reload nftables
+ tags: nftables
+ ansible.builtin.systemd_service:
+ name: nftables
+ state: reloaded

ansible/roles/nftables/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: Install nftables
+ ansible.builtin.apt:
+ cache_valid_time: 86400
+ name: nftables
+ tags:
+ - nftables
+
+- name: create config dir
+ ansible.builtin.file:
+ path: /etc/ooni/nftables/tcp
+ state: directory
+ owner: root
+ group: root
+ mode: 0755
+ tags:
+ - nftables
+
+- name: allow SSH
+ ansible.builtin.blockinfile:
+ path: /etc/ooni/nftables/tcp/22.nft
+ create: yes
+ block: |
+ add rule inet filter input tcp dport 22 counter accept comment "Incoming SSH"
+ tags:
+ - nftables
+
+- name: Overwrite nftables.conf
+ ansible.builtin.template:
+ src: templates/nftables.conf
+ dest: /etc/nftables.conf
+ mode: 0755
+ owner: root
+ notify:
+ - Reload nftables
+ tags:
+ - nftables
+
+- name: enable nftables service
+ ansible.builtin.systemd_service:
+ name: nftables
+ enabled: yes
+ state: started
+ tags:
+ - nftables