Anonymous credentials draft endpoints

fastpath/clickhouse_init.sql

@@ -38,7 +38,12 @@ CREATE TABLE IF NOT EXISTS default.fastpath
     `blocking_type` String,
     `test_helper_address` LowCardinality(String),
     `test_helper_type` LowCardinality(String),
-    `ooni_run_link_id` Nullable(UInt64)
+    `ooni_run_link_id` Nullable(UInt64),
+    `is_verified` Int8,
+    `nym` Nullable(String),
+    `zkp_request` Nullable(String),
+    `age_range` Nullable(String),
+    `msm_range` Nullable(String),
 )
 ENGINE = ReplacingMergeTree
 ORDER BY (measurement_start_time, report_id, input)
@@ -202,4 +207,3 @@ CREATE TABLE IF NOT EXISTS default.fingerprints_http
 )
 ENGINE = EmbeddedRocksDB
 PRIMARY KEY name;
-

fastpath/fastpath/core.py

@@ -1616,7 +1616,7 @@ def flag_measurements_with_wrong_date(msm: dict, msmt_uid: str, scores: dict) ->
         scores["msg"] = "Measurement start time too old"
 
 def write_measurement_to_disk(msm_tup) -> None:
-    """Write this measurement to disk so that it can be 
+    """Write this measurement to disk so that it can be
     processed by the measurement uploader
 
     Args:
@@ -1633,7 +1633,7 @@ def write_measurement_to_disk(msm_tup) -> None:
     msmtdir = spooldir / "incoming" / dirname
     msmtdir.mkdir(parents=True, exist_ok=True)
 
-    try: 
+    try:
         msmt_f_tmp = msmtdir / f"{msmt_uid}.post.tmp"
         msmt_f_tmp.write_bytes(data)
         msmt_f = msmtdir / f"{msmt_uid}.post"
@@ -1654,7 +1654,14 @@ def process_measurement(msm_tup, buffer_writes=False) -> None:
         assert msmt_uid
         if measurement is None:
             measurement = ujson.loads(msm_jstr)
-        if sorted(measurement.keys()) == ["content", "format"]:
+
+        is_verified = g(measurement, 'is_verified', False)
+        nym = g(measurement, 'nym')
+        zkp_request = g(measurement, 'zkp_request')
+        age_range = g(measurement, 'age_range')
+        msm_range = g(measurement, 'msm_range')
+
+        if "content" in measurement and "format" in measurement:
             measurement = unwrap_msmt(measurement)
         rid = measurement.get("report_id")
         inp = measurement.get("input")
@@ -1742,6 +1749,11 @@ def process_measurement(msm_tup, buffer_writes=False) -> None:
             test_helper_address,
             test_helper_type,
             ooni_run_link_id,
+            is_verified,
+            nym,
+            zkp_request,
+            age_range,
+            msm_range,
             buffer_writes=buffer_writes,
         )
 

fastpath/fastpath/db.py

@@ -10,7 +10,7 @@
 from datetime import datetime
 from textwrap import dedent
 from urllib.parse import urlparse
-from typing import List, Tuple, Dict, Optional
+from typing import List, Tuple, Dict, Optional, Any
 import logging
 
 try:
@@ -209,6 +209,11 @@ def clickhouse_upsert_summary(
     test_helper_address: str,
     test_helper_type: str,
     ooni_run_link_id: Optional[int],
+    is_verified: bool,
+    zkp_request: Optional[str],
+    nym: Optional[str],
+    age_range: Optional[Tuple[int, int]],
+    msm_range: Optional[Tuple[int, int]],
     buffer_writes=False,
 ) -> None:
     """Insert a row in the fastpath table. Overwrite an existing one."""
@@ -224,6 +229,10 @@ def nn(features: dict, k: str) -> str:
     def tf(v: bool) -> str:
         return "t" if v else "f"
 
+    def serialize_optional(x : Optional[Any]) -> Optional[str]:
+        """Serialize to string if not None, return None otherwise"""
+        return ujson.dumps(x) if x is not None else None
+
     test_name = msm.get("test_name", None) or ""
     input_, domain = extract_input_domain(msm, test_name)
     asn = int(msm["probe_asn"][2:])  # AS123
@@ -257,6 +266,11 @@ def tf(v: bool) -> str:
         test_helper_address=test_helper_address,
         test_helper_type=test_helper_type,
         ooni_run_link_id=ooni_run_link_id,
+        is_verified=tf(is_verified),
+        nym=nym,
+        zkp_request=zkp_request,
+        age_range=serialize_optional(age_range),
+        msm_range=serialize_optional(msm_range)
     )
 
     if buffer_writes:

fastpath/fastpath/tests/test_functional_nodb.py

@@ -154,6 +154,11 @@ def test_score_web_connectivity_bug_610_2(fprints):
             "test_helper_address": "https://0.th.ooni.org",
             "test_helper_type": "https",
             "ooni_run_link_id": None,
+            "is_verified" : "f",
+            "nym" : None,
+            "zkp_request" : None,
+            "age_range" : None,
+            "msm_range" : None,
         }
     ]
 
@@ -201,6 +206,11 @@ def test_score_browser_web(fprints):
             "test_runtime": 0.35740000000037253,
             "test_start_time": datetime.datetime(2023, 3, 20, 18, 26, 35),
             "test_version": "0.1.0",
+            "is_verified" : "f",
+            "nym" : None,
+            "zkp_request" : None,
+            "age_range" : None,
+            "msm_range" : None,
         },
     ]
 
@@ -252,6 +262,11 @@ def test_score_openvpn():
             "test_helper_address": "",
             "test_helper_type": "",
             "ooni_run_link_id": None,
+            "is_verified" : "f",
+            "nym" : None,
+            "zkp_request" : None,
+            "age_range" : None,
+            "msm_range" : None,
         }
     ]
 

fastpath/makefile

@@ -80,18 +80,18 @@ docker:
 
 # Runs docker in foreground, useful for checking errors in the image before it runs
 docker-fg:
-	docker compose --profile default up --build 
+	docker compose --profile default up --build
 
-# Runs both fastpath and the testing clickhous. 
+# Runs both fastpath and the testing clickhous.
 # Mind the fastpath configuration in fastpath.conf
 docker-all: docker-clickhouse
 	echo "Waiting for clickhouse..."
-	sleep 4 
-	docker compose --profile default up --build -d 
+	sleep 4
+	docker compose --profile default up --build -d
 
 # Turns off every service
 docker-down:
-	docker compose --profile all down 
+	docker compose --profile all down
 
 # If you need to test the fastpath locally you can use this rule to spawn the clickhouse database
 # locally and then use `make docker` or `make docker-fg` to start the fastpath container. Ex:
@@ -109,15 +109,15 @@ docker-login:
 
 # Get logs from the fastpath docker service
 docker-logs:
-	docker compose logs fastpath -f 
+	docker compose logs fastpath -f
 
-# Get logs for a specified service. Example: 
+# Get logs for a specified service. Example:
 # `make docker-logs-for args="clickhouse-server"`
 docker-logs-for:
-	docker compose logs $(args) -f 
+	docker compose logs $(args) -f
 
 # Used for actually building the fastpath docker image
-docker-build: 
+docker-build:
 # We need to use tar -czh to resolve the common dir symlink
 	tar -czh . | docker build \
 		--build-arg BUILD_LABEL=${BUILD_LABEL} \

ooniapi/common/src/common/alembic/versions/7e28b5d17a7f_add_server_state_table_for_anonymous_.py

@@ -0,0 +1,71 @@
+"""Add server state table for anonymous credentials
+
+Revision ID: 7e28b5d17a7f
+Revises: 8e7ecea5c2f5
+Create Date: 2025-10-08 10:36:55.796144
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy.schema as sc
+
+
+# revision identifiers, used by Alembic.
+revision: str = "7e28b5d17a7f"
+down_revision: Union[str, None] = "8e7ecea5c2f5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+# ---
+ooniprobe_server_state_id_seq = sc.Sequence("ooniprobe_server_state_id_seq", start=1)
+ooniprobe_manifest_id_seq = sc.Sequence("ooniprobe_manifest_id_seq", start=1)
+
+
+def upgrade() -> None:
+    op.execute(sc.CreateSequence(ooniprobe_server_state_id_seq))
+
+    op.create_table(
+        "ooniprobe_server_state",
+        sa.Column(
+            "id",
+            sa.String(),
+            nullable=False,
+            server_default=ooniprobe_server_state_id_seq.next_value(),
+            primary_key=True,
+        ),
+        sa.Column("date_created", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("secret_key", sa.String(), nullable=False),
+        sa.Column("public_parameters", sa.String(), nullable=False),
+    )
+
+    op.execute(sc.CreateSequence(ooniprobe_manifest_id_seq))
+    op.create_table(
+        "ooniprobe_manifest",
+        sa.Column(
+            "version",
+            sa.String(),
+            nullable=False,
+            server_default=ooniprobe_manifest_id_seq.next_value(),
+            primary_key=True,
+        ),
+        sa.Column("date_created", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("nym_scope", sa.String(), nullable=False),
+        sa.Column("submission_policy", sa.JSON(), nullable=False),
+
+        sa.Column(
+            "server_state_id",
+            sa.String(),
+            sa.ForeignKey("ooniprobe_server_state.id"),
+            nullable=False,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("ooniprobe_manifest")
+    op.drop_table("ooniprobe_server_state")
+    op.execute(sc.DropSequence(ooniprobe_server_state_id_seq))
+    op.execute(sc.DropSequence(ooniprobe_manifest_id_seq))

ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql

@@ -65,6 +65,11 @@ ENGINE = MergeTree
 ORDER BY (report_id, input)
 SETTINGS index_granularity = 8192;
 
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `is_verified` Int8;
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `nym` Nullable(String);
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `zkp_request` Nullable(String);
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `age_range` Nullable(String);
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `msm_range` Nullable(String);
 CREATE TABLE IF NOT EXISTS default.event_detector_changepoints
 (
     `probe_asn` UInt32,
@@ -106,4 +111,4 @@ PARTITION BY toYYYYMM(ts)
 ORDER BY (probe_asn, probe_cc, ts, domain)
 SETTINGS index_granularity = 8192;
 
-ALTER TABLE default.event_detector_changepoints ADD COLUMN `block_type` String;
+ALTER TABLE default.event_detector_changepoints ADD COLUMN `block_type` String;

ooniapi/services/ooniprobe/Dockerfile

@@ -1,5 +1,5 @@
 # Python builder
-FROM python:3.11-bookworm as builder
+FROM python:3.11-bookworm AS builder
 ARG BUILD_LABEL=docker
 
 WORKDIR /build
@@ -14,10 +14,10 @@ RUN find /build -type f -name '._*' -delete
 
 RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL
 
-RUN hatch build
+RUN make build
 
 ### Actual image running on the host
-FROM python:3.11-bookworm as runner
+FROM python:3.11-bookworm AS runner
 
 WORKDIR /app
 

ooniapi/services/ooniprobe/pyproject.toml

@@ -28,7 +28,8 @@ dependencies = [
   "fastapi-utils[all] ~= 0.8.0",
   "zstd ~= 1.5.7.2",
   "boto3 ~= 1.39.3",
-  "boto3-stubs[s3] ~= 1.39.3"
+  "boto3-stubs[s3] ~= 1.39.3",
+  "ooniauth-py @ https://github.com/ooni/userauth/raw/refs/heads/release-wheels/ooniauth-py/wheels/ooniauth_py-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl"
 ]
 
 readme = "README.md"
@@ -82,6 +83,13 @@ test-cov = "pytest -s --full-trace --log-level=INFO  --log-cli-level=INFO -v --s
 cov-report = ["coverage report"]
 cov = ["test-cov", "cov-report"]
 
+# Allows specifyng dependencies as a direct reference like an URL
+# or a file path.
+# Required by ooniauth-py
+# See: https://hatch.pypa.io/1.13/config/metadata/#allowing-direct-references
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.pytest.ini_options]
 addopts = ["--import-mode=importlib"]
 

ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py

@@ -6,7 +6,7 @@
 import geoip2.database
 
 from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, Session
 
 from clickhouse_driver import Client as Clickhouse
 
@@ -15,6 +15,7 @@
 
 from .common.config import Settings
 from .common.dependencies import get_settings
+from .models import OONIProbeManifest
 
 
 SettingsDep: TypeAlias = Annotated[Settings, Depends(get_settings)]
@@ -23,14 +24,16 @@
 def get_postgresql_session(settings: SettingsDep):
     engine = create_engine(settings.postgresql_url)
     SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
     db = SessionLocal()
     try:
         yield db
     finally:
         db.close()
 
 
+PostgresSessionDep = Annotated[Session, Depends(get_postgresql_session)]
+
+
 def get_cc_reader(settings: SettingsDep):
     db_path = Path(settings.geoip_db_dir, "cc.mmdb")
     return geoip2.database.Reader(db_path)
@@ -64,3 +67,12 @@ def get_s3_client() -> S3Client:
 
 
 S3ClientDep = Annotated[S3Client, Depends(get_s3_client)]
+
+
+def get_latest_manifest(session: PostgresSessionDep) -> OONIProbeManifest:
+    manifest = OONIProbeManifest.get_latest(session)
+    assert manifest is not None, "Uninitialized `OONIProbeServerState` table"
+    return manifest
+
+
+LatestManifestDep = Annotated[OONIProbeManifest, Depends(get_latest_manifest)]