connectors-ci: report path should always start with airbyte-ci/ (#2…

…9030) * make report path always start with airbyte-ci * revert report path in orchestrator * add more test cases * bump version
airbytehq · Aug 3, 2023 · 9a4b158 · 9a4b158
1 parent b189548
commit 9a4b158
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 10 deletions.
diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py
@@ -1,15 +1,18 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 import os
 from typing import Optional
 
 VALID_REGISTRIES = ["oss", "cloud"]
 REGISTRIES_FOLDER = "registries/v0"
 REPORT_FOLDER = "generated_reports"
 
-NIGHTLY_FOLDER = "airbyte-ci-internal/connectors/test/nightly_builds/master"
+NIGHTLY_FOLDER = "airbyte-ci/connectors/test/nightly_builds/master"
 NIGHTLY_COMPLETE_REPORT_FILE_NAME = "complete.json"
 NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME = "output.json"
 NIGHTLY_GHA_WORKFLOW_ID = "connector_nightly_builds_dagger.yml"
-CI_TEST_REPORT_PREFIX = "airbyte-ci-internal/connectors/test"
+CI_TEST_REPORT_PREFIX = "airbyte-ci/connectors/test"
 CI_MASTER_TEST_OUTPUT_REGEX = f".*master.*output.json$"
 
 CONNECTOR_REPO_NAME = "airbytehq/airbyte"

diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -380,6 +380,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 
 | Version | PR                                                        | Description                                                                                  |
 |---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------|
+| 0.4.2   | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`.                                 |
 | 0.4.1   | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands.                           |
 | 0.4.0   | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI                                                             |
 | 0.3.2   | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull.                                                |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/utils.py b/airbyte-ci/connectors/pipelines/pipelines/utils.py
@@ -41,6 +41,7 @@
 DIFF_FILTER = "MADRT"  # Modified, Added, Deleted, Renamed, Type changed
 IGNORED_FILE_EXTENSIONS = [".md"]
 ALL_CONNECTOR_DEPENDENCIES = [(connector, connector.get_local_dependency_paths()) for connector in get_all_connectors_in_repo()]
+STATIC_REPORT_PREFIX = "airbyte-ci"
 
 
 # This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented
@@ -512,7 +513,12 @@ def render_report_output_prefix(ctx: click.Context) -> str:
         sanitized_branch = slugify(git_branch.replace("/", "_"))
 
         # get the command name for the current context, if a group then prepend the parent command name
-        cmd = ctx.command_path.replace(" ", "/") if ctx.command_path else None
+        if ctx.command_path:
+            cmd_components = ctx.command_path.split(" ")
+            cmd_components[0] = STATIC_REPORT_PREFIX
+            cmd = "/".join(cmd_components)
+        else:
+            cmd = None
 
         path_values = [
             cmd,

diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "0.4.1"
+version = "0.4.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

diff --git a/airbyte-ci/connectors/pipelines/tests/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py
@@ -21,7 +21,7 @@
                     "ci_job_key": None,
                 },
             ),
-            "my/command/path/my_ci_context/my_branch/my_pipeline_start_timestamp/my_git_revision",
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_context/my_branch/my_pipeline_start_timestamp/my_git_revision",
         ),
         (
             mock.MagicMock(
@@ -34,7 +34,7 @@
                     "ci_job_key": "my_ci_job_key",
                 },
             ),
-            "my/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision",
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision",
         ),
         (
             mock.MagicMock(
@@ -47,7 +47,7 @@
                     "ci_job_key": "my_ci_job_key",
                 },
             ),
-            "my/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision",
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision",
         ),
         (
             mock.MagicMock(
@@ -60,7 +60,7 @@
                     "ci_job_key": "my_ci_job_key",
                 },
             ),
-            "my/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision",
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision",
         ),
         (
             mock.MagicMock(
@@ -73,7 +73,7 @@
                     "ci_job_key": "my_ci_job_key",
                 },
             ),
-            "my/command/path/my_ci_job_key/my_branch_with_slashes/my_pipeline_start_timestamp/my_git_revision",
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashes/my_pipeline_start_timestamp/my_git_revision",
         ),
         (
             mock.MagicMock(
@@ -86,7 +86,33 @@
                     "ci_job_key": "my_ci_job_key",
                 },
             ),
-            "my/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision",
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision",
+        ),
+        (
+            mock.MagicMock(
+                command_path="airbyte-ci command path",
+                obj={
+                    "git_branch": "my_branch/with/slashes#and!special@characters",
+                    "git_revision": "my_git_revision",
+                    "pipeline_start_timestamp": "my_pipeline_start_timestamp",
+                    "ci_context": "my_ci_context",
+                    "ci_job_key": "my_ci_job_key",
+                },
+            ),
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision",
+        ),
+        (
+            mock.MagicMock(
+                command_path="airbyte-ci-internal command path",
+                obj={
+                    "git_branch": "my_branch/with/slashes#and!special@characters",
+                    "git_revision": "my_git_revision",
+                    "pipeline_start_timestamp": "my_pipeline_start_timestamp",
+                    "ci_context": "my_ci_context",
+                    "ci_job_key": "my_ci_job_key",
+                },
+            ),
+            f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision",
         ),
     ],
 )