connectors-ci: make unhandled error lead to failed report (#28789)

airbytehq · Aug 3, 2023 · 519b71e · 519b71e
1 parent 137e510
commit 519b71e
Show file tree

Hide file tree

Showing 6 changed files with 35 additions and 6 deletions.
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -378,6 +378,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 
 | Version | PR                                                        | Description                                                                                  |
 |---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------|
+| 0.3.2   | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull.                                              |
 | 0.3.1   | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped                                |
 | 0.3.0   | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution                                |
 | 0.2.3   | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command                                    |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py
@@ -368,7 +368,7 @@ def skipped_steps(self) -> List[StepResult]:  # noqa D102
 
     @property
     def success(self) -> bool:  # noqa D102
-        return len(self.failed_steps) == 0
+        return len(self.failed_steps) == 0 and (len(self.skipped_steps) > 0 or len(self.successful_steps) > 0)
 
     @property
     def run_duration(self) -> timedelta:  # noqa D102

diff --git a/airbyte-ci/connectors/pipelines/pipelines/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/contexts.py
@@ -15,15 +15,15 @@
 import yaml
 from anyio import Path
 from asyncer import asyncify
+from connector_ops.utils import Connector
+from dagger import Client, Directory, Secret
+from github import PullRequest
 from pipelines import hacks
 from pipelines.actions import secrets
 from pipelines.bases import CIContext, ConnectorReport, Report
 from pipelines.github import update_commit_status_check
 from pipelines.slack import send_message_to_webhook
 from pipelines.utils import AIRBYTE_REPO_URL, METADATA_FILE_NAME, format_duration, sanitize_gcs_credentials
-from connector_ops.utils import Connector
-from dagger import Client, Directory, Secret
-from github import PullRequest
 
 
 class ContextState(Enum):

diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py
@@ -8,12 +8,12 @@
 
 import anyio
 import asyncer
+from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage
 from pipelines.bases import ConnectorReport, StepResult
 from pipelines.contexts import ConnectorContext
 from pipelines.pipelines.metadata import MetadataValidation
 from pipelines.tests import java_connectors, python_connectors
 from pipelines.tests.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck
-from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage
 
 LANGUAGE_MAPPING = {
     "run_all_tests": {

diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "0.3.1"
+version = "0.3.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py
@@ -39,3 +39,31 @@ async def test_run_with_timeout(self, test_context):
         assert step_result.stderr == timed_out_step_result.stderr
         assert step_result.output_artifact == timed_out_step_result.output_artifact
         assert step.retry_count == step.max_retries + 1
+
+
+class TestReport:
+    @pytest.fixture
+    def test_context(self, mocker):
+        return mocker.Mock()
+
+    def test_report_failed_if_it_has_no_step_result(self, test_context):
+        report = bases.Report(test_context, [])
+        assert not report.success
+        report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.FAILURE)])
+        assert not report.success
+
+        report = bases.Report(
+            test_context, [bases.StepResult(None, bases.StepStatus.FAILURE), bases.StepResult(None, bases.StepStatus.SUCCESS)]
+        )
+        assert not report.success
+
+        report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.SUCCESS)])
+        assert report.success
+
+        report = bases.Report(
+            test_context, [bases.StepResult(None, bases.StepStatus.SUCCESS), bases.StepResult(None, bases.StepStatus.SKIPPED)]
+        )
+        assert report.success
+
+        report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.SKIPPED)])
+        assert report.success