Merge branch 'v5.0.0-rc.0' into fastapi

* v5.0.0-rc.0:
  Add simple redirector
  Tidy up the layout of the analysis viewer
  Add an observations viewer
  Get rid of all the dataviz that isn't the analysis visualizer
  Release/5.0.0 alpha3 (#81)
  Offset analysis schedule by 6 hours
  Add support for temporal cloud (#79)
  fix: support for sorting network_events using transaction_id (#51)
  Optimize performance of table writers and refactor table model (#74)
  Improvements related to deployment (#69)
  Add .codecov file
  Update jsonl sync example
  Setup workflow to publish ooni data docs (#73)
  OONI Pipeline v5 alpha (#64)
  Fix codecov (#62)
  Temporal workflows (#61)
  OONI Data, OONI Pipeline split (#60)
  Add support for caching netinfodb

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+ status:
+ project:
+ default:
+ target: 80
+ patch:
+ default:
+ target: 80

.github/workflows/build_docs.yml

@@ -0,0 +1,50 @@
+name: build docs
+on: push
+
+jobs:
+ build_docs:
+ runs-on: "ubuntu-20.04"
+ steps:
+ - name: Check out repository code
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Build docs
+ run: make docs
+
+ - name: Get current git ref
+ id: rev_parse
+ run: echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+ - name: Checkout ooni/docs
+ uses: actions/checkout@v2
+ with:
+ repository: "ooni/docs"
+ ssh-key: ${{ secrets.OONI_DOCS_DEPLOYKEY }}
+ path: "ooni-docs"
+
+ - name: Update docs
+ run: |
+ mkdir -p ooni-docs/src/content/docs/data/
+ cp -R dist/docs/* ooni-docs/src/content/docs/data/
+
+ - name: Check for conflicting slugs
+ run: |
+ cat ooni-docs/src/content/docs/data/*.md \
+ | grep "^slug:" | awk -F':' '{gsub(/^ +/, "", $2); print $2}' | sort | uniq -c \
+ | awk '{if ($1 > 1) { print "duplicate slug for: " $2; exit 1}}'
+
+ - name: Print the lines of the generated docs
+ run: wc -l ooni-docs/src/content/docs/data/*
+
+ - name: Commit changes
+ # Only push the docs update when we are in master
+ # if: github.ref == 'refs/heads/master'
+ run: |
+ cd ooni-docs
+ git config --global user.email "github+backend@ooni.org"
+ git config --global user.name "OONI Github Actions Bot"
+ git add .
+ git commit -m "auto: update ooni/data docs to ${{ steps.rev_parse.outputs.COMMIT_HASH }}" || echo "No changes to commit"
+ git push origin

.github/workflows/test_oonidata.yml

@@ -0,0 +1,50 @@
+name: test oonidata
+on: push
+jobs:
+ run_tests:
+ name: ${{ matrix.os }} / ${{ matrix.python-version }}
+ runs-on: ${{ matrix.os }}-latest
+ strategy:
+ matrix:
+ os: [Ubuntu]
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install hatch
+ run: pip install hatch
+
+ - name: Set up datadir cache
+ uses: actions/cache@v3
+ with:
+ path: oonidata/tests/data/datadir/
+ key: oonidata-tests-data-datadir
+
+ - name: Set up measurements cache
+ uses: actions/cache@v3
+ with:
+ path: oonidata/tests/data/measurements/
+ key: oonidata-tests-data-measurements
+
+ - name: Set up raw_measurements cache
+ uses: actions/cache@v3
+ with:
+ path: oonidata/tests/data/raw_measurements/
+ key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+ - name: Run all tests
+ run: hatch run cov
+ working-directory: ./oonidata/
+
+ - name: Upload coverage to codecov
+ uses: codecov/codecov-action@v3
+ with:
+ flags: oonidata
+ working-directory: ./oonidata/
+ env:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/test_oonipipeline.yml

@@ -0,0 +1,62 @@
+name: test oonipipeline
+on: push
+jobs:
+ run_tests:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v3
+
+ # Contrary to oonidata, which is a tool for end users, oonipipeline
+ # targets a single python version to minimize the likelyhood of breakage
+ # in production.
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+
+ - name: Install hatch
+ run: pip install hatch
+
+ - name: Set up datadir cache
+ uses: actions/cache@v3
+ with:
+ path: oonipipeline/tests/data/datadir/
+ key: oonipipeline-tests-data-datadir
+
+ - name: Set up measurements cache
+ uses: actions/cache@v3
+ with:
+ path: oonipipeline/tests/data/measurements/
+ key: oonipipeline-tests-data-measurements
+
+ - name: Set up raw_measurements cache
+ uses: actions/cache@v3
+ with:
+ path: oonipipeline/tests/data/raw_measurements/
+ key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+ - name: Install clickhouse
+ run: |
+ sudo apt-get install -y apt-transport-https ca-certificates dirmngr
+ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
+ echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
+ /etc/apt/sources.list.d/clickhouse.list
+ sudo apt-get update
+ sudo apt-get install -y clickhouse-server clickhouse-client
+
+ - name: Install temporal
+ run: |
+ curl -sSf https://temporal.download/cli.sh | sh
+ echo "$HOME/.temporalio/bin" >> $GITHUB_PATH
+
+ - name: Run all tests
+ run: hatch run cov -v
+ working-directory: ./oonipipeline/
+
+ - name: Upload coverage to codecov
+ uses: codecov/codecov-action@v3
+ with:
+ flags: oonipipeline
+ working-directory: ./oonipipeline/
+ env:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -1,12 +1,13 @@
 __pycache__
-/.coverage
-/coverage.xml
-/tests/data/datadir/*
-/tests/data/raw_measurements/*
-/tests/data/measurements/*
+.coverage
+.coverage.*
+coverage.xml
+
+.ipynb_checkpoints/
+
 /dist
-/datadir
 /output
 /attic
 /prof
-.ipynb_checkpoints/
+
+.DS_Store

.vscode/settings.json

@@ -3,6 +3,7 @@
  "python.testing.pytestArgs": ["tests"],
  "python.testing.unittestEnabled": false,
  "python.testing.pytestEnabled": true,
+ "python.defaultInterpreterPath": "${workspaceFolder}/oonipipeline/.venv",
  "[python]": {
  "editor.defaultFormatter": "ms-python.black-formatter"
  },

Makefile

@@ -0,0 +1,7 @@
+docs:
+ ./scripts/build_docs.sh
+
+clean:
+ rm -rf dist/
+
+.PHONY: docs clean