From 78b53f064efa22fc15aa4c845538d1a53baa2d42 Mon Sep 17 00:00:00 2001 From: decfox Date: Fri, 19 Jul 2024 01:05:45 +0530 Subject: [PATCH 1/2] feat: add oonifindings A/B tests --- testshift/LICENSE.txt | 26 ++++++++ testshift/README.md | 21 ++++++ testshift/pyproject.toml | 95 ++++++++++++++++++++++++++++ testshift/src/testshift/__about__.py | 1 + testshift/src/testshift/__init__.py | 0 testshift/tests/__init__.py | 0 testshift/tests/test_oonifindings.py | 32 ++++++++++ 7 files changed, 175 insertions(+) create mode 100644 testshift/LICENSE.txt create mode 100644 testshift/README.md create mode 100644 testshift/pyproject.toml create mode 100644 testshift/src/testshift/__about__.py create mode 100644 testshift/src/testshift/__init__.py create mode 100644 testshift/tests/__init__.py create mode 100644 testshift/tests/test_oonifindings.py diff --git a/testshift/LICENSE.txt b/testshift/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/testshift/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/testshift/README.md b/testshift/README.md new file mode 100644 index 00000000..ef31d54c --- /dev/null +++ b/testshift/README.md @@ -0,0 +1,21 @@ +# testshift + +[![PyPI - Version](https://img.shields.io/pypi/v/testshift.svg)](https://pypi.org/project/testshift) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/testshift.svg)](https://pypi.org/project/testshift) + +----- + +**Table of Contents** + +- [Installation](#installation) +- [License](#license) + +## Installation + +```console +pip install testshift +``` + +## License + +`testshift` is distributed under the terms of the [OONI] (https://github.com/ooni/license/blob/master/software/LICENSE.md) license. diff --git a/testshift/pyproject.toml b/testshift/pyproject.toml new file mode 100644 index 00000000..13882f4a --- /dev/null +++ b/testshift/pyproject.toml @@ -0,0 +1,95 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "testshift" +dynamic = ["version"] +description = '' + +dependencies = [ + "httpx ~= 0.26.0", +] + +readme = "README.md" +requires-python = ">=3.11" +license = "BSD-3-Clause" +keywords = [] +authors = [ + { name = "OONI", email = "contact@ooni.org" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Documentation = "https://docs.ooni.org" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/testshift/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/testshift"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-asyncio", +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.types] +dependencies = [ + "mypy>=1.0.0", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/testshift tests}" + +[tool.coverage.run] +source_pkgs = ["testshift", "tests"] +branch = true +parallel = true +omit = [ + "src/testshift/common/*", + "src/testshift/__about__.py" +] + +[tool.coverage.paths] +testshift = ["src/testshift", "*/testshift/src/testshift"] +tests = ["tests", "*/testshift/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/testshift/src/testshift/__about__.py b/testshift/src/testshift/__about__.py new file mode 100644 index 00000000..f102a9ca --- /dev/null +++ b/testshift/src/testshift/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/testshift/src/testshift/__init__.py b/testshift/src/testshift/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testshift/tests/__init__.py b/testshift/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testshift/tests/test_oonifindings.py b/testshift/tests/test_oonifindings.py new file mode 100644 index 00000000..fc999335 --- /dev/null +++ b/testshift/tests/test_oonifindings.py @@ -0,0 +1,32 @@ +from typing import Dict + +import httpx + +LEGACY_HOST = "https://backend-hel.ooni.org" +ACTIVE_HOST = "https://api.dev.ooni.io" + + +def check_response_keys(legacy_response: Dict, active_response: Dict): + return legacy_response.keys() == active_response.keys() + + +def test_oonifindings(): + with httpx.Client(base_url=LEGACY_HOST) as legacy_client, httpx.Client(base_url=ACTIVE_HOST) as active_client: + legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=false") + active_response = active_client.get("api/v1/incidents/search?only_mine=false") + legacy_incidents, active_incidents = legacy_response.json(), active_response.json() + assert len(legacy_incidents) == len(active_incidents) + for idx in range(len(legacy_incidents)): + assert check_response_keys(legacy_incidents[idx], active_incidents[idx]) + + legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=true") + active_response = active_client.get("api/v1/incidents/search?only_mine=true") + legacy_incidents, active_incidents = legacy_response.json(), active_response.json() + assert len(legacy_incidents) == len(active_incidents) + for idx in range(len(legacy_incidents)): + assert check_response_keys(legacy_incidents[idx], active_incidents[idx]) + + sample_incident = "" + legacy_response = legacy_client.get(f"api/v1/incidents/show/{sample_incident}") + active_response = legacy_client.get(f"api/v1/incidents/show/{sample_incident}") + assert check_response_keys(legacy_response.json(), active_response.json()) From 53a25cf1442a1457c84c84c1c850ece5ee5163df Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 15 Aug 2024 01:22:32 +0530 Subject: [PATCH 2/2] feat: A/B test oonifindings with new and legacy deployments --- testshift/tests/test_oonifindings.py | 42 ++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/testshift/tests/test_oonifindings.py b/testshift/tests/test_oonifindings.py index fc999335..b095d2d7 100644 --- a/testshift/tests/test_oonifindings.py +++ b/testshift/tests/test_oonifindings.py @@ -2,31 +2,49 @@ import httpx -LEGACY_HOST = "https://backend-hel.ooni.org" +LEGACY_HOST = "https://api.ooni.io" ACTIVE_HOST = "https://api.dev.ooni.io" -def check_response_keys(legacy_response: Dict, active_response: Dict): - return legacy_response.keys() == active_response.keys() +# NOTE: The new API has been updated to include one new response +# field which we do not use and hence default to an empty value. +# +# `creator_account_id`: "" +def check_search_response_keys(legacy_response: Dict, active_response: Dict): + legacy_keys = list(legacy_response.keys()) + active_keys = list(active_response.keys()) + + assert len(active_keys) == len(legacy_keys) + 1 + + active_keys.remove("creator_account_id") + return sorted(active_keys) == sorted(legacy_keys) def test_oonifindings(): with httpx.Client(base_url=LEGACY_HOST) as legacy_client, httpx.Client(base_url=ACTIVE_HOST) as active_client: legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=false") active_response = active_client.get("api/v1/incidents/search?only_mine=false") - legacy_incidents, active_incidents = legacy_response.json(), active_response.json() + legacy_incidents, active_incidents = legacy_response.json()["incidents"], active_response.json()["incidents"] + assert len(legacy_incidents) == len(active_incidents) + for idx in range(len(legacy_incidents)): - assert check_response_keys(legacy_incidents[idx], active_incidents[idx]) + assert check_search_response_keys(legacy_incidents[idx], active_incidents[idx]) legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=true") active_response = active_client.get("api/v1/incidents/search?only_mine=true") - legacy_incidents, active_incidents = legacy_response.json(), active_response.json() + legacy_incidents, active_incidents = legacy_response.json()["incidents"], active_response.json()["incidents"] + assert len(legacy_incidents) == len(active_incidents) + for idx in range(len(legacy_incidents)): - assert check_response_keys(legacy_incidents[idx], active_incidents[idx]) - - sample_incident = "" - legacy_response = legacy_client.get(f"api/v1/incidents/show/{sample_incident}") - active_response = legacy_client.get(f"api/v1/incidents/show/{sample_incident}") - assert check_response_keys(legacy_response.json(), active_response.json()) + assert check_search_response_keys(legacy_incidents[idx], active_incidents[idx]) + + incident_id = "330022197701" + legacy_response = legacy_client.get(f"api/v1/incidents/show/{incident_id}") + active_response = active_client.get(f"api/v1/incidents/show/{incident_id}") + + legacy_incident = legacy_response.json()["incident"] + active_incident = active_response.json()["incident"] + + assert check_search_response_keys(legacy_incident, active_incident)