From 022c0542ae8d81df98363c41e5af30d7c84c3544 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 27 Jan 2023 11:37:25 +0100 Subject: [PATCH 1/3] qa-engine: define cloud eligibility --- .../ci_connector_ops/qa_engine/models.py | 3 + .../ci_connector_ops/qa_engine/validations.py | 24 ++++++ tools/ci_connector_ops/setup.py | 2 +- .../tests/test_qa_engine/test_validations.py | 74 ++++++++++++++++++- 4 files changed, 101 insertions(+), 2 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py b/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py index 7fef17500db3c..8831d6e456911 100644 --- a/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py +++ b/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py @@ -3,6 +3,7 @@ # +from datetime import datetime from enum import Enum from typing import List @@ -32,6 +33,8 @@ class ConnectorQAReport(BaseModel): number_of_connections: int number_of_users: int sync_success_rate: float + is_eligible_for_cloud: bool + report_generation_datetime: datetime class QAReport(BaseModel): connectors_qa_report: List[ConnectorQAReport] diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py b/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py index 8a04e8b91bb8a..af6047cbffeb1 100644 --- a/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py +++ b/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py @@ -3,6 +3,9 @@ # +from datetime import datetime +from typing import Iterable + import pandas as pd import requests @@ -10,6 +13,12 @@ from .inputs import OSS_CATALOG from .models import ConnectorQAReport, QAReport +TRUTHY_COLUMNS_TO_BE_ELIGIBLE = [ + "documentation_is_available", + "is_appropriate_for_cloud_use", + "latest_build_is_successful" +] + class QAReportGenerationError(Exception): pass @@ -20,6 +29,14 @@ def url_is_reachable(url: str) -> bool: def is_appropriate_for_cloud_use(definition_id: str) -> bool: return definition_id not in INAPPROPRIATE_FOR_CLOUD_USE_CONNECTORS +def is_eligible_for_cloud(connector_qa_data: pd.Series) -> bool: + if connector_qa_data["is_on_cloud"]: + return False + return all([ + connector_qa_data[col] + for col in TRUTHY_COLUMNS_TO_BE_ELIGIBLE + ]) + def get_qa_report(enriched_catalog: pd.DataFrame) -> pd.DataFrame: """Perform validation steps on top of the enriched catalog. Adds the following columns: @@ -51,6 +68,9 @@ def get_qa_report(enriched_catalog: pd.DataFrame) -> pd.DataFrame: qa_report["number_of_users"] = 0 # TODO, tracked in https://github.com/airbytehq/airbyte/issues/21721 qa_report["sync_success_rate"] = .0 # TODO, tracked in https://github.com/airbytehq/airbyte/issues/21721 + qa_report["is_eligible_for_cloud"] = qa_report.apply(is_eligible_for_cloud, axis="columns") + qa_report["report_generation_datetime"] = datetime.utcnow() + # Only select dataframe columns defined in the ConnectorQAReport model. qa_report= qa_report[[field.name for field in ConnectorQAReport.__fields__.values()]] # Validate the report structure with pydantic QAReport model. @@ -58,3 +78,7 @@ def get_qa_report(enriched_catalog: pd.DataFrame) -> pd.DataFrame: if len(qa_report) != len(OSS_CATALOG): raise QAReportGenerationError("The QA report does not contain all the connectors defined in the OSS catalog.") return qa_report + +def get_connectors_eligible_for_cloud(qa_report: pd.DataFrame) -> Iterable[ConnectorQAReport]: + for _, row in qa_report[qa_report["is_eligible_for_cloud"]].iterrows(): + yield ConnectorQAReport(**row) diff --git a/tools/ci_connector_ops/setup.py b/tools/ci_connector_ops/setup.py index 1b85d559a3bec..50cc3f0fe40b3 100644 --- a/tools/ci_connector_ops/setup.py +++ b/tools/ci_connector_ops/setup.py @@ -23,7 +23,7 @@ setup( - version="0.1.6", + version="0.1.8", name="ci_connector_ops", description="Packaged maintained by the connector operations team to perform CI for connectors", author="Airbyte", diff --git a/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py b/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py index c20dc12f4b8a3..28529e4fce55f 100644 --- a/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py +++ b/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py @@ -6,7 +6,7 @@ import pandas as pd import pytest -from ci_connector_ops.qa_engine import inputs, enrichments, models, validations +from ci_connector_ops.qa_engine import enrichments, inputs, models, validations @pytest.fixture def enriched_catalog() -> pd.DataFrame: @@ -32,3 +32,75 @@ def test_report_generation_error(enriched_catalog, mocker): mocker.patch.object(validations, "url_is_reachable", mocker.Mock(return_value=True)) with pytest.raises(validations.QAReportGenerationError): return validations.get_qa_report(enriched_catalog.sample(10)) + +@pytest.mark.parametrize( + "connector_qa_data, expected_to_be_eligible", + [ + ( + pd.Series({ + "is_on_cloud": False, + "documentation_is_available": True, + "is_appropriate_for_cloud_use": True, + "latest_build_is_successful": True + }), + True + ), + ( + pd.Series({ + "is_on_cloud": True, + "documentation_is_available": True, + "is_appropriate_for_cloud_use": True, + "latest_build_is_successful": True + }), + False + ), + ( + pd.Series({ + "is_on_cloud": True, + "documentation_is_available": False, + "is_appropriate_for_cloud_use": False, + "latest_build_is_successful": False + }), + False + ), + ( + pd.Series({ + "is_on_cloud": False, + "documentation_is_available": False, + "is_appropriate_for_cloud_use": True, + "latest_build_is_successful": True + }), + False + ), + ( + pd.Series({ + "is_on_cloud": False, + "documentation_is_available": True, + "is_appropriate_for_cloud_use": False, + "latest_build_is_successful": True + }), + False + ), + ( + pd.Series({ + "is_on_cloud": False, + "documentation_is_available": True, + "is_appropriate_for_cloud_use": True, + "latest_build_is_successful": False + }), + False + ) + ] +) +def test_is_eligible_for_cloud(connector_qa_data: pd.Series, expected_to_be_eligible: bool): + assert validations.is_eligible_for_cloud(connector_qa_data) == expected_to_be_eligible + +def test_get_connectors_eligible_for_cloud(qa_report: pd.DataFrame): + qa_report["is_eligible_for_cloud"] = True + connectors_eligible_for_cloud = list(validations.get_connectors_eligible_for_cloud(qa_report)) + assert len(qa_report) == len(connectors_eligible_for_cloud) + assert all([c.is_eligible_for_cloud for c in connectors_eligible_for_cloud]) + + qa_report["is_eligible_for_cloud"] = False + connectors_eligible_for_cloud = list(validations.get_connectors_eligible_for_cloud(qa_report)) + assert len(connectors_eligible_for_cloud) == 0 From 2bb724c9ac11e0b51b4e0febb20c317cec0d124d Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 27 Jan 2023 15:54:51 +0100 Subject: [PATCH 2/3] fix version --- tools/ci_connector_ops/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_connector_ops/setup.py b/tools/ci_connector_ops/setup.py index 50cc3f0fe40b3..f8dced1a5444c 100644 --- a/tools/ci_connector_ops/setup.py +++ b/tools/ci_connector_ops/setup.py @@ -23,7 +23,7 @@ setup( - version="0.1.8", + version="0.1.7", name="ci_connector_ops", description="Packaged maintained by the connector operations team to perform CI for connectors", author="Airbyte", From cb9424af20e241c6fd7eb05c276c9372a18b2147 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 27 Jan 2023 16:07:53 +0100 Subject: [PATCH 3/3] is_eligible_for_cloud -> is_eligible_for_promotion_to_cloud --- .../ci_connector_ops/qa_engine/models.py | 2 +- .../ci_connector_ops/qa_engine/validations.py | 6 +++--- .../tests/test_qa_engine/test_validations.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py b/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py index 8831d6e456911..37eb14be1246a 100644 --- a/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py +++ b/tools/ci_connector_ops/ci_connector_ops/qa_engine/models.py @@ -33,7 +33,7 @@ class ConnectorQAReport(BaseModel): number_of_connections: int number_of_users: int sync_success_rate: float - is_eligible_for_cloud: bool + is_eligible_for_promotion_to_cloud: bool report_generation_datetime: datetime class QAReport(BaseModel): diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py b/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py index af6047cbffeb1..b168b59e1f516 100644 --- a/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py +++ b/tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py @@ -29,7 +29,7 @@ def url_is_reachable(url: str) -> bool: def is_appropriate_for_cloud_use(definition_id: str) -> bool: return definition_id not in INAPPROPRIATE_FOR_CLOUD_USE_CONNECTORS -def is_eligible_for_cloud(connector_qa_data: pd.Series) -> bool: +def is_eligible_for_promotion_to_cloud(connector_qa_data: pd.Series) -> bool: if connector_qa_data["is_on_cloud"]: return False return all([ @@ -68,7 +68,7 @@ def get_qa_report(enriched_catalog: pd.DataFrame) -> pd.DataFrame: qa_report["number_of_users"] = 0 # TODO, tracked in https://github.com/airbytehq/airbyte/issues/21721 qa_report["sync_success_rate"] = .0 # TODO, tracked in https://github.com/airbytehq/airbyte/issues/21721 - qa_report["is_eligible_for_cloud"] = qa_report.apply(is_eligible_for_cloud, axis="columns") + qa_report["is_eligible_for_promotion_to_cloud"] = qa_report.apply(is_eligible_for_promotion_to_cloud, axis="columns") qa_report["report_generation_datetime"] = datetime.utcnow() # Only select dataframe columns defined in the ConnectorQAReport model. @@ -80,5 +80,5 @@ def get_qa_report(enriched_catalog: pd.DataFrame) -> pd.DataFrame: return qa_report def get_connectors_eligible_for_cloud(qa_report: pd.DataFrame) -> Iterable[ConnectorQAReport]: - for _, row in qa_report[qa_report["is_eligible_for_cloud"]].iterrows(): + for _, row in qa_report[qa_report["is_eligible_for_promotion_to_cloud"]].iterrows(): yield ConnectorQAReport(**row) diff --git a/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py b/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py index 28529e4fce55f..a56a5678e009c 100644 --- a/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py +++ b/tools/ci_connector_ops/tests/test_qa_engine/test_validations.py @@ -92,15 +92,15 @@ def test_report_generation_error(enriched_catalog, mocker): ) ] ) -def test_is_eligible_for_cloud(connector_qa_data: pd.Series, expected_to_be_eligible: bool): - assert validations.is_eligible_for_cloud(connector_qa_data) == expected_to_be_eligible +def test_is_eligible_for_promotion_to_cloud(connector_qa_data: pd.Series, expected_to_be_eligible: bool): + assert validations.is_eligible_for_promotion_to_cloud(connector_qa_data) == expected_to_be_eligible def test_get_connectors_eligible_for_cloud(qa_report: pd.DataFrame): - qa_report["is_eligible_for_cloud"] = True + qa_report["is_eligible_for_promotion_to_cloud"] = True connectors_eligible_for_cloud = list(validations.get_connectors_eligible_for_cloud(qa_report)) assert len(qa_report) == len(connectors_eligible_for_cloud) - assert all([c.is_eligible_for_cloud for c in connectors_eligible_for_cloud]) + assert all([c.is_eligible_for_promotion_to_cloud for c in connectors_eligible_for_cloud]) - qa_report["is_eligible_for_cloud"] = False + qa_report["is_eligible_for_promotion_to_cloud"] = False connectors_eligible_for_cloud = list(validations.get_connectors_eligible_for_cloud(qa_report)) assert len(connectors_eligible_for_cloud) == 0