Oonirun v2 1

ooniapi/common/src/common/alembic/versions/b860eb79750f_add_targets_name_and_inputs_extra_.py

@@ -0,0 +1,36 @@
+"""Add targets_name and inputs_extra columns to oonirun_nettest
+
+Revision ID: b860eb79750f
+Revises: 8e7ecea5c2f5
+Create Date: 2025-05-21 15:44:32.959349
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "b860eb79750f"
+down_revision: Union[str, None] = "8e7ecea5c2f5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "oonirun_nettest", sa.Column("targets_name", sa.String(), nullable=True)
+    )
+    op.add_column(
+        "oonirun_nettest", sa.Column("inputs_extra", sa.ARRAY(sa.JSON()), nullable=True)
+    )
+    op.drop_column("oonirun_nettest", "backend_options")
+
+
+def downgrade() -> None:
+    op.drop_column("oonirun_nettest", "targets_name")
+    op.drop_column("oonirun_nettest", "inputs_extra")
+
+    op.add_column("oonirun_nettest", sa.Column("backend_options", sa.ARRAY(sa.JSON()), nullable=True))

ooniapi/common/src/common/postgresql.py

@@ -9,4 +9,5 @@ class Base(DeclarativeBase):
         Dict[str, Any]: sa.JSON,
         List[str]: sa.JSON,
         Dict[str, str]: sa.JSON,
+        List[Dict[str, Any]] : sa.ARRAY(sa.JSON())
     }

ooniapi/services/ooniprobe/src/ooniprobe/prio.py → ooniapi/common/src/common/prio.py

@@ -1,3 +1,4 @@
+
 """
 OONI Probe Services API - reactive URL prioritization
 
@@ -24,8 +25,8 @@
 from typing import List, Tuple
 import logging
 
-from .common.clickhouse_utils import query_click
-from .common.metrics import timer
+from .clickhouse_utils import query_click
+from .metrics import timer
 
 from clickhouse_driver import Client as Clickhouse
 import sqlalchemy as sa
@@ -154,4 +155,4 @@ def generate_test_list(
 
     if debug:
         return out, entries, prio_rules
-    return out, (), ()
+    return out, (), ()

ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py

@@ -20,7 +20,7 @@
 from ...common.auth import create_jwt, decode_jwt, jwt
 from ...common.config import Settings
 from ...common.utils import setnocacheresponse
-from ...prio import generate_test_list
+from ...common.prio import generate_test_list
 
 router = APIRouter(prefix="/v1")
 

ooniapi/services/ooniprobe/tests/test_prio.py

@@ -1,4 +1,4 @@
-from ooniprobe import prio
+from ooniprobe.common import prio
 
 
 def test_prio():

ooniapi/services/oonirun/pyproject.toml

@@ -58,7 +58,7 @@ packages = ["src/oonirun"]
 artifacts = ["BUILD_LABEL"]
 
 [tool.hatch.envs.default]
-dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql", "pytest-asyncio"]
+dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql", "pytest-asyncio", "pytest-docker"]
 path = ".venv/"
 
 [tool.hatch.envs.default.scripts]

ooniapi/services/oonirun/src/oonirun/dependencies.py

@@ -1,16 +1,20 @@
-from functools import lru_cache
 from typing import Annotated
 
 from fastapi import Depends
 
 from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, Session
+
+from clickhouse_driver import Client as Clickhouse
 
 from .common.config import Settings
 from .common.dependencies import get_settings
 
 
-def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)]):
+DependsSettings = Annotated[Settings, Depends(get_settings)]
+
+
+def get_postgresql_session(settings: DependsSettings):
     engine = create_engine(settings.postgresql_url)
     SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
@@ -19,3 +23,17 @@ def get_postgresql_session(settings: Annotated[Settings, Depends(get_settings)])
         yield db
     finally:
         db.close()
+
+
+DependsPostgresSession = Annotated[Session, Depends(get_postgresql_session)]
+
+
+def get_clickhouse_session(settings: DependsSettings):
+    db = Clickhouse.from_url(settings.clickhouse_url)
+    try:
+        yield db
+    finally:
+        db.disconnect()
+
+
+DependsClickhouseClient = Annotated[Clickhouse, Depends(get_clickhouse_session)]

ooniapi/services/oonirun/src/oonirun/main.py

@@ -11,12 +11,18 @@
 from . import models
 from .routers import v2
 
-from .dependencies import get_postgresql_session
+from .dependencies import (
+    DependsPostgresSession,
+    DependsClickhouseClient,
+    DependsSettings,
+)
 from .common.dependencies import get_settings
 from .common.version import get_build_label, get_pkg_version
 from .common.version import get_build_label, get_pkg_version
 from .common.metrics import mount_metrics
+from .common.clickhouse_utils import query_click
 
+log = logging.getLogger(__name__)
 
 pkg_name = "oonirun"
 
@@ -63,10 +69,23 @@ class HealthStatus(BaseModel):
 
 @app.get("/health")
 async def health(
-    settings=Depends(get_settings),
-    db=Depends(get_postgresql_session),
+    settings: DependsSettings,
+    db: DependsPostgresSession,
+    clickhouse: DependsClickhouseClient,
 ):
     errors = []
+
+    try:
+        query = """
+        SELECT COUNT()
+        FROM fastpath
+        WHERE measurement_start_time < NOW() AND measurement_start_time > NOW() - INTERVAL 3 HOUR
+        """
+        query_click(db=clickhouse, query=query, query_params={})
+    except Exception as e:
+        errors.append("clickhouse_error")
+        log.error(e)
+
     try:
         db.query(models.OONIRunLink).limit(1).all()
     except Exception as exc:

ooniapi/services/oonirun/src/oonirun/models.py

@@ -57,7 +57,8 @@ class OONIRunLinkNettest(Base):
     test_name: Mapped[str] = mapped_column()
     inputs: Mapped[List[str]] = mapped_column(nullable=True)
     options: Mapped[Dict[str, Any]] = mapped_column(nullable=True)
-    backend_options: Mapped[Dict[str, Any]] = mapped_column(nullable=True)
 
     is_background_run_enabled_default: Mapped[bool] = mapped_column(default=False)
     is_manual_run_enabled_default: Mapped[bool] = mapped_column(default=False)
+    targets_name: Mapped[str] = mapped_column(nullable=True)
+    inputs_extra: Mapped[List[Dict[str, Any]]] = mapped_column(nullable=True)