airbyte-ci: send publish failures to a dedicated slack channel (#42849)

.github/workflows/publish_connectors.yml

@@ -81,11 +81,11 @@ jobs:
           body: '{ "trigger": "down", "status": "HASISSUES" }'
 
   notify-failure-slack-channel:
-    name: "Notify Slack Channel on Build Failures"
+    name: "Notify Slack Channel on Publish Failures"
     runs-on: ubuntu-latest
     needs:
       - publish_connectors
-    if: ${{ failure() && github.ref == 'refs/heads/master' }}
+    if: ${{ always() && contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v4
@@ -95,16 +95,16 @@ jobs:
         env:
           AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Publish to OSS Build Failure Slack Channel
-        uses: abinoda/slack-action@master
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+      - name: Send publish failures to connector-publish-failures channel
+        id: slack
+        uses: slackapi/slack-github-action@v1.26.0
         with:
-          args: >-
-            {\"channel\":\"C056HGD1QSW\", \"blocks\":[
-            {\"type\":\"divider\"},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Publish Connector Failed! :bangbang: \n\n\"}},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
-            {\"type\":\"divider\"}]}
+          # This data can be any valid JSON from a previous step in the GitHub Action
+          payload: |
+            {
+              "channel": "#connector-publish-failures",
+              "username": "Connectors CI/CD Bot",
+              "text": "🚨 Publish workflow failed:\n ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}

airbyte-ci/connectors/pipelines/README.md

@@ -571,8 +571,8 @@ For Python connectors, sets the `airbyte-cdk` dependency in `pyproject.toml` and
 
 #### Arguments
 
-| Argument      | Description                                             |
-| ------------- | ------------------------------------------------------- |
+| Argument      | Description                                                               |
+| ------------- | ------------------------------------------------------------------------- |
 | `CDK_VERSION` | CDK version constraint to set (default to `^{most_recent_patch_version}`) |
 
 #### Notes
@@ -772,7 +772,8 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 ## Changelog
 
 | Version | PR                                                         | Description                                                                                                                  |
-|---------| ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------|
+| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.26.0  | [#42849](https://github.com/airbytehq/airbyte/pull/42849)  | Send publish failures messages to `#connector-publish-failures`                                                              |
 | 4.25.4  | [#42463](https://github.com/airbytehq/airbyte/pull/42463) | Add validation before live test runs                                                                                         |
 | 4.25.3  | [#42437](https://github.com/airbytehq/airbyte/pull/42437)  | Ugrade-cdk: Update to work with Python connectors using poetry                                                               |
 | 4.25.2  | [#42077](https://github.com/airbytehq/airbyte/pull/42077)  | Live/regression tests: add status check for regression test runs                                                             |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py

@@ -7,7 +7,7 @@
 
 from dagger import Container, Platform
 from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
-from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase, apply_airbyte_docker_labels
+from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.dagger.actions.python.common import apply_python_development_overrides, with_python_connector_installed
 from pipelines.models.steps import StepResult

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py

@@ -56,7 +56,6 @@ def __init__(
         pipeline_start_timestamp: Optional[int] = None,
         ci_context: Optional[str] = None,
         slack_webhook: Optional[str] = None,
-        reporting_slack_channel: Optional[str] = None,
         pull_request: Optional[PullRequest.PullRequest] = None,
         should_save_report: bool = True,
         code_tests_only: bool = False,
@@ -88,7 +87,6 @@ def __init__(
             pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None.
             ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None.
             slack_webhook (Optional[str], optional): The slack webhook to send messages to. Defaults to None.
-            reporting_slack_channel (Optional[str], optional): The slack channel to send messages to. Defaults to None.
             pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None.
             code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False.
             use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False.
@@ -131,7 +129,6 @@ def __init__(
             pipeline_start_timestamp=pipeline_start_timestamp,
             ci_context=ci_context,
             slack_webhook=slack_webhook,
-            reporting_slack_channel=reporting_slack_channel,
             pull_request=pull_request,
             ci_report_bucket=ci_report_bucket,
             ci_gcp_credentials=ci_gcp_credentials,
@@ -258,11 +255,8 @@ async def __aexit__(
         await asyncify(update_commit_status_check)(**self.github_commit_status)
 
         if self.should_send_slack_message:
-            # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel
-            await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook)  # type: ignore
+            # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook
+            await asyncify(send_message_to_webhook)(self.create_slack_message(), self.get_slack_channels(), self.slack_webhook)  # type: ignore
 
         # Supress the exception if any
         return True
-
-    def create_slack_message(self) -> str:
-        raise NotImplementedError

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py

@@ -56,13 +56,6 @@
     type=click.STRING,
     envvar="SLACK_WEBHOOK",
 )
-@click.option(
-    "--slack-channel",
-    help="The Slack webhook URL to send notifications to.",
-    type=click.STRING,
-    envvar="SLACK_CHANNEL",
-    default="#connector-publish-updates",
-)
 @click.option(
     "--python-registry-token",
     help="Access token for python registry",
@@ -93,7 +86,6 @@ async def publish(
     metadata_service_bucket_name: str,
     metadata_service_gcs_credentials: Secret,
     slack_webhook: str,
-    slack_channel: str,
     python_registry_token: Secret,
     python_registry_url: str,
     python_registry_check_url: str,
@@ -119,7 +111,6 @@ async def publish(
                 docker_hub_username=Secret("docker_hub_username", ctx.obj["secret_stores"]["in_memory"]),
                 docker_hub_password=Secret("docker_hub_password", ctx.obj["secret_stores"]["in_memory"]),
                 slack_webhook=slack_webhook,
-                reporting_slack_channel=slack_channel,
                 ci_report_bucket=ctx.obj["ci_report_bucket_name"],
                 report_output_prefix=ctx.obj["report_output_prefix"],
                 is_local=ctx.obj["is_local"],

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py

@@ -4,12 +4,12 @@
 
 """Module declaring context related classes."""
 
-from typing import Optional
+from typing import List, Optional
 
 import asyncclick as click
 from github import PullRequest
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.consts import ContextState
+from pipelines.consts import PUBLISH_FAILURE_SLACK_CHANNEL, PUBLISH_UPDATES_SLACK_CHANNEL, ContextState
 from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles
 from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX
 from pipelines.helpers.utils import format_duration
@@ -29,7 +29,6 @@ def __init__(
         docker_hub_password: Secret,
         ci_gcp_credentials: Secret,
         slack_webhook: str,
-        reporting_slack_channel: str,
         ci_report_bucket: str,
         report_output_prefix: str,
         is_local: bool,
@@ -78,7 +77,6 @@ def __init__(
             pipeline_start_timestamp=pipeline_start_timestamp,
             ci_context=ci_context,
             slack_webhook=slack_webhook,
-            reporting_slack_channel=reporting_slack_channel,
             ci_gcp_credentials=ci_gcp_credentials,
             should_save_report=True,
             use_local_cdk=use_local_cdk,
@@ -108,6 +106,12 @@ def docker_image_tag(self) -> str:
         else:
             return metadata_tag
 
+    def get_slack_channels(self) -> List[str]:
+        if self.state in [ContextState.FAILURE, ContextState.ERROR]:
+            return [PUBLISH_UPDATES_SLACK_CHANNEL, PUBLISH_FAILURE_SLACK_CHANNEL]
+        else:
+            return [PUBLISH_UPDATES_SLACK_CHANNEL]
+
     def create_slack_message(self) -> str:
 
         docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags"

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

@@ -13,9 +13,8 @@
 from functools import cached_property
 from pathlib import Path
 from textwrap import dedent
-from typing import Any, ClassVar, Dict, List, Optional, Set
+from typing import ClassVar, List, Optional, Set
 
-import dagger
 import requests  # type: ignore
 import semver
 import yaml  # type: ignore

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py

@@ -7,7 +7,6 @@
 from typing import TYPE_CHECKING
 
 import asyncer
-import click
 import dagger
 import toml
 from pipelines.airbyte_ci.test.models import deserialize_airbyte_ci_config

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -68,6 +68,9 @@
 AIRBYTE_SUBMODULE_DIR_NAME = "airbyte-submodule"
 MANUAL_PIPELINE_STATUS_CHECK_OVERRIDE_PREFIXES = ["Regression Tests"]
 
+PUBLISH_UPDATES_SLACK_CHANNEL = "#connector-publish-updates"
+PUBLISH_FAILURE_SLACK_CHANNEL = "#connector-publish-failures"
+
 
 class CIContext(str, Enum):
     """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"]."""

airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py

@@ -1,19 +1,28 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+from __future__ import annotations
 
 import json
+import typing
 
 import requests
 from pipelines import main_logger
 
+if typing.TYPE_CHECKING:
+    from typing import List
 
-def send_message_to_webhook(message: str, channel: str, webhook: str) -> requests.Response:
-    payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message}
-    response = requests.post(webhook, data={"payload": json.dumps(payload)})
 
-    # log if the request failed, but don't fail the pipeline
-    if not response.ok:
-        main_logger.error(f"Failed to send message to slack webhook: {response.text}")
+def send_message_to_webhook(message: str, channels: List[str], webhook: str) -> List[requests.Response]:
+    responses = []
+    for channel in channels:
+        channel = channel[1:] if channel.startswith("#") else channel
+        payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message}
+        response = requests.post(webhook, data={"payload": json.dumps(payload)})
 
-    return response
+        # log if the request failed, but don't fail the pipeline
+        if not response.ok:
+            main_logger.error(f"Failed to send message to slack webhook: {response.text}")
+        responses.append(response)
+
+    return responses

airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py

@@ -78,7 +78,6 @@ def __init__(
         ci_context: Optional[str] = None,
         is_ci_optional: bool = False,
         slack_webhook: Optional[str] = None,
-        reporting_slack_channel: Optional[str] = None,
         pull_request: Optional[PullRequest.PullRequest] = None,
         ci_report_bucket: Optional[str] = None,
         ci_gcp_credentials: Optional[Secret] = None,
@@ -104,7 +103,6 @@ def __init__(
             ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None.
             is_ci_optional (bool, optional): Whether the CI is optional. Defaults to False.
             slack_webhook (Optional[str], optional): Slack webhook to send messages to. Defaults to None.
-            reporting_slack_channel (Optional[str], optional): Slack channel to send messages to. Defaults to None.
             pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None.
         """
         self.pipeline_name = pipeline_name
@@ -122,7 +120,6 @@ def __init__(
         self.state = ContextState.INITIALIZED
         self.is_ci_optional = is_ci_optional
         self.slack_webhook = slack_webhook
-        self.reporting_slack_channel = reporting_slack_channel
         self.pull_request = pull_request
         self.logger = logging.getLogger(self.pipeline_name)
         self._dagger_client = None
@@ -200,7 +197,7 @@ def github_commit_status(self) -> dict:
 
     @property
     def should_send_slack_message(self) -> bool:
-        return self.slack_webhook is not None and self.reporting_slack_channel is not None
+        return self.slack_webhook is not None
 
     @property
     def has_dagger_cloud_token(self) -> bool:
@@ -267,6 +264,9 @@ def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, i
     def create_slack_message(self) -> str:
         raise NotImplementedError()
 
+    def get_slack_channels(self) -> List[str]:
+        raise NotImplementedError()
+
     async def __aenter__(self) -> PipelineContext:
         """Perform setup operation for the PipelineContext.
 
@@ -284,10 +284,8 @@ async def __aenter__(self) -> PipelineContext:
         self.logger.info("Caching the latest CDK version...")
         await asyncify(update_commit_status_check)(**self.github_commit_status)
         if self.should_send_slack_message:
-            # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel
-            await asyncify(send_message_to_webhook)(
-                self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook  # type: ignore
-            )
+            # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook
+            await asyncify(send_message_to_webhook)(self.create_slack_message(), self.get_slack_channels(), self.slack_webhook)  # type: ignore
         return self
 
     @staticmethod
@@ -343,9 +341,9 @@ async def __aexit__(
 
         await asyncify(update_commit_status_check)(**self.github_commit_status)
         if self.should_send_slack_message:
-            # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel
+            # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook
             await asyncify(send_message_to_webhook)(
-                self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook  # type: ignore
+                self.create_slack_message(), self.get_slack_channels(), self.slack_webhook  # type: ignore
             )
         # supress the exception if it was handled
         return True

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.25.4"
+version = "4.26.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py

@@ -31,7 +31,7 @@ def connector_context(dagger_client):
 @pytest.mark.parametrize("use_local_cdk", [True, False])
 async def test_apply_python_development_overrides(connector_context, use_local_cdk):
     connector_context.use_local_cdk = use_local_cdk
-    fake_connector_container = connector_context.dagger_client.container().from_("airbyte/python-connector-base:1.0.0")
+    fake_connector_container = connector_context.dagger_client.container().from_("airbyte/python-connector-base:2.0.0")
     before_override_pip_freeze = await fake_connector_container.with_exec(["pip", "freeze"]).stdout()
 
     assert "airbyte-cdk" not in before_override_pip_freeze.splitlines(), "The base image should not have the airbyte-cdk installed."

airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py

@@ -27,7 +27,11 @@ def latest_cdk_version():
 def python_connector_with_setup_not_latest_cdk(all_connectors):
     for connector in all_connectors:
         if (
-            connector.metadata.get("connectorBuildOptions", {}).get("baseImage", False)
+            connector.metadata.get("connectorBuildOptions", False)
+            # We want to select a connector with a base image version >= 2.0.0 to use Python 3.10
+            and not connector.metadata.get("connectorBuildOptions", {})
+            .get("baseImage", "")
+            .startswith("docker.io/airbyte/python-connector-base:1.")
             and connector.language == "python"
             and connector.code_directory.joinpath("setup.py").exists()
         ):