Consolidate dev and prod deployments and switch th back to DO (#96)

Changes:
* Add support for creating digital ocean droplets for test helpers
* Switch networking config to NAT gateway less setup

Fixes:
* #92
* #91
* #93

Checklist for doing it:
* [x] Create terraform module for deploying test helpers to digital
ocean
* [x] Tweak ECS task sizes to reduce instance consumption
* [x] Drop ECS cluster for test helpers
* [x] Setup direct load balancer rules to address services based on
hostname
* [x] Drop test helper monitoring on AWS in monitoring host
* [x] Add support for SAN in ACM certificates by creating new module
* [x] Setup nginx based load balancing for test helpers
* [x] Refactor EC2 instance deployment to use cloud-init

Since AWS costs are too high for the test helpers with IPv6 support, we
switch them back to digital ocean.

How this is implemented is by adding a new rule to the oonibackend proxy
that acts as a load balancer towards the test helpers on digital ocean.
The reason to do this is so that we don't have to complicate the TLS
setup by having to do certificate provisioning on the test helpers, but
rather are able to keep it in AWS.

Moreover by having a single entry point to the test helpers it means we
can implement a cache which works across all the test helper backends,
instead of having a per-test helper caching layer.

What is missing is adding the rules that perform routing on a per domain
basis to the load balancer config.

---------

Co-authored-by: decfox <mehul@openobservatory.org>

ansible/roles/prometheus/templates/prometheus.yml

@@ -99,7 +99,9 @@ scrape_configs:
  password: '{{ prometheus_metrics_password_dev }}'
  static_configs:
  - targets:
- - oohelperd.th.dev.ooni.io
+ - ooniauth.dev.ooni.io
+ - oonirun.dev.ooni.io
+ - ooniprobe.dev.ooni.io
 
  - job_name: 'ooniapi-services-prod'
  scrape_interval: 5s
@@ -110,11 +112,25 @@ scrape_configs:
  password: '{{ prometheus_metrics_password_prod }}'
  static_configs:
  - targets:
- - ooniauth.api.prod.ooni.io
- - oonirun.api.prod.ooni.io
- - ooniprobe.api.prod.ooni.io
- - oohelperd.th.prod.ooni.io
+ - ooniauth.prod.ooni.io
+ - oonirun.prod.ooni.io
+ - ooniprobe.prod.ooni.io
 
+ - job_name: 'oonith-prod'
+ scrape_interval: 5s
+ scheme: http
+ metrics_path: "/metrics"
+ basic_auth:
+ username: 'prom'
+ password: '{{ prometheus_metrics_password_prod }}'
+ static_configs:
+ - targets:
+ - 0.do.th.prod.ooni.io:9001
+ - 0.do.th.prod.ooni.io
+ - 1.do.th.prod.ooni.io:9001
+ - 1.do.th.prod.ooni.io
+ - 2.do.th.prod.ooni.io
+ - 2.do.th.prod.ooni.io:9001
 
  - job_name: 'ooni-web'
  scrape_interval: 5m

ansible/roles/prometheus/vars/main.yml

@@ -28,6 +28,8 @@ blackbox_jobs:
  - "https://2.th.ooni.org/"
  - "https://3.th.ooni.org/"
  - "https://4.th.ooni.org/"
+ - "https://5.th.ooni.org/"
+ - "https://6.th.ooni.org/"
 
  - name: "ooni collector"
  module: "ooni_collector_ok"
@@ -73,21 +75,44 @@ blackbox_jobs:
  module: "http_2xx"
  targets:
  - "https://api.ooni.io/api/v1/measurements"
+ - "https://api.ooni.org/api/v1/measurements"
 
  - name: "ooni API test-list urls"
  module: "https_2xx_json_meta"
  targets:
  - "https://api.ooni.io/api/v1/test-list/urls?country_code=US"
+ - "https://api.ooni.org/api/v1/test-list/urls?country_code=US"
 
  - name: "ooni API test-helpers"
  module: "https_2xx_json"
  targets:
  - "https://api.ooni.io/api/v1/test-helpers"
+ - "https://api.ooni.org/api/v1/test-helpers"
 
  - name: "ooni API priv global overview"
  module: "https_2xx_json"
  targets:
  - "https://api.ooni.io/api/_/global_overview"
+ - "https://api.ooni.org/api/_/global_overview"
+
+ # Note: this always returns true by design
+ - name: "OONI API check_report_id"
+ module: "https_2xx_json"
+ targets:
+ - "https://api.ooni.io/api/_/check_report_id?report_id=RANDOM"
+ - "https://api.ooni.org/api/_/check_report_id?report_id=RANDOM"
+
+ - name: "OONI API raw_measurement"
+ module: "https_2xx_json"
+ targets:
+ - "https://api.ooni.io/api/v1/raw_measurement?measurement_uid=20240924151005.116855_IT_httpinvalidrequestline_f63463817af9eebe"
+ - "https://api.ooni.org/api/v1/raw_measurement?measurement_uid=20240924151005.116855_IT_httpinvalidrequestline_f63463817af9eebe"
+
+ - name: "OONI Run v2 API"
+ module: "https_2xx_json"
+ targets:
+ - "https://api.ooni.org/api/v2/oonirun/links/10009"
+ - "https://api.ooni.org/api/v2/oonirun/links/10009/revisions"
 
  # end of API #