Draft Common Message Queue

.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml

@@ -51,6 +51,7 @@ body:
         - cohere
         - common-compat
         - common-io
+        - common-messaging
         - common-sql
         - databricks
         - datadog

INSTALL

@@ -232,13 +232,14 @@ or dependencies that are necessary to enable the feature in an editable build.
 airbyte, alibaba, amazon, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.flink,
 apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka, apache.kylin, apache.livy,
 apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana, atlassian.jira, celery, cloudant,
-cncf.kubernetes, cohere, common.compat, common.io, common.sql, databricks, datadog, dbt.cloud,
-dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp, github, google, grpc,
-hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure, microsoft.mssql, microsoft.psrp,
-microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie,
-oracle, pagerduty, papermill, pgvector, pinecone, postgres, presto, qdrant, redis, salesforce,
-samba, segment, sendgrid, sftp, singularity, slack, smtp, snowflake, sqlite, ssh, standard, tableau,
-telegram, teradata, trino, vertica, weaviate, yandex, ydb, zendesk
+cncf.kubernetes, cohere, common.compat, common.io, common.messaging, common.sql, databricks,
+datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp,
+github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure,
+microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas,
+openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector, pinecone, postgres,
+presto, qdrant, redis, salesforce, samba, segment, sendgrid, sftp, singularity, slack, smtp,
+snowflake, sqlite, ssh, standard, tableau, telegram, teradata, trino, vertica, weaviate, yandex,
+ydb, zendesk
 
 # END PROVIDER EXTRAS HERE
 

contributing-docs/12_airflow_dependencies_and_extras.rst

@@ -180,13 +180,14 @@ or dependencies that are necessary to enable the feature in editable build.
 airbyte, alibaba, amazon, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.flink,
 apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka, apache.kylin, apache.livy,
 apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana, atlassian.jira, celery, cloudant,
-cncf.kubernetes, cohere, common.compat, common.io, common.sql, databricks, datadog, dbt.cloud,
-dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp, github, google, grpc,
-hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure, microsoft.mssql, microsoft.psrp,
-microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie,
-oracle, pagerduty, papermill, pgvector, pinecone, postgres, presto, qdrant, redis, salesforce,
-samba, segment, sendgrid, sftp, singularity, slack, smtp, snowflake, sqlite, ssh, standard, tableau,
-telegram, teradata, trino, vertica, weaviate, yandex, ydb, zendesk
+cncf.kubernetes, cohere, common.compat, common.io, common.messaging, common.sql, databricks,
+datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp,
+github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure,
+microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas,
+openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector, pinecone, postgres,
+presto, qdrant, redis, salesforce, samba, segment, sendgrid, sftp, singularity, slack, smtp,
+snowflake, sqlite, ssh, standard, tableau, telegram, teradata, trino, vertica, weaviate, yandex,
+ydb, zendesk
 
   .. END PROVIDER EXTRAS HERE
 

dev/breeze/doc/images/output_build-docs.txt

@@ -1 +1 @@
-7945acec5c69cdf8a296229be4eb11ca
+1464900883acc757de0795666f0f6e7f

dev/breeze/doc/images/output_release-management_add-back-references.txt

@@ -1 +1 @@
-03f7d04a09488fcd7c28c13d0653ff7e
+28ca3febf77042587cb889cc72053872

dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt

@@ -1 +1 @@
-6806d4e405c59f8ee43796ae638b1308
+871ba1422cbc18888edcd35f8d731bf5

dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt

@@ -1 +1 @@
-07b221762eed9e3344466b445ca6668b
+e9d4cb942e2fd7387e303bdfe0698f2b

dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt

@@ -1 +1 @@
-bb317d3fef1d85908c6ffa160924d36a
+e751c14da28c25ad10b6d79ea2be4bfb

dev/breeze/doc/images/output_release-management_publish-docs.txt

@@ -1 +1 @@
-c9f176661908b6f7fc5795ceaedfa5e1
+923a2a604d01a06a2c76d1be9b8e2bb0

dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt

@@ -1 +1 @@
-c77360bfe9bd3d86bfc4e6a763969fb6
+2b7b93625b6e3adc20712223eb230494

dev/breeze/src/airflow_breeze/global_constants.py

@@ -597,13 +597,13 @@ def get_airflow_extras():
     {
         "python-version": "3.9",
         "airflow-version": "2.9.3",
-        "remove-providers": "cloudant fab edge",
+        "remove-providers": "cloudant common.messaging fab edge",
         "run-tests": "true",
     },
     {
         "python-version": "3.9",
         "airflow-version": "2.10.4",
-        "remove-providers": "cloudant fab",
+        "remove-providers": "cloudant common.messaging fab",
         "run-tests": "true",
     },
 ]

dev/breeze/tests/test_selective_checks.py

@@ -718,7 +718,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             ("providers/amazon/src/airflow/providers/amazon/__init__.py",),
             {
                 "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
-                "common.compat common.sql exasol fab ftp google http imap microsoft.azure "
+                "common.compat common.messaging common.sql exasol fab ftp google http imap microsoft.azure "
                 "mongo mysql openlineage postgres salesforce ssh teradata",
                 "all-python-versions": "['3.9']",
                 "all-python-versions-list-as-string": "3.9",
@@ -735,7 +735,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                 "upgrade-to-newer-dependencies": "false",
                 "run-amazon-tests": "true",
                 "core-test-types-list-as-string": "Always",
-                "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,common.compat,common.sql,exasol,fab,ftp,http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] Providers[google]",
+                "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,common.compat,common.messaging,common.sql,exasol,fab,ftp,http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] Providers[google]",
                 "needs-mypy": "true",
                 "mypy-checks": "['mypy-providers']",
             },
@@ -770,7 +770,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             ("providers/amazon/src/airflow/providers/amazon/file.py",),
             {
                 "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
-                "common.compat common.sql exasol fab ftp google http imap microsoft.azure "
+                "common.compat common.messaging common.sql exasol fab ftp google http imap microsoft.azure "
                 "mongo mysql openlineage postgres salesforce ssh teradata",
                 "all-python-versions": "['3.9']",
                 "all-python-versions-list-as-string": "3.9",
@@ -787,7 +787,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                 "run-kubernetes-tests": "false",
                 "upgrade-to-newer-dependencies": "false",
                 "core-test-types-list-as-string": "Always",
-                "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,common.compat,common.sql,exasol,fab,ftp,http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] Providers[google]",
+                "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,common.compat,common.messaging,common.sql,exasol,fab,ftp,http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] Providers[google]",
                 "needs-mypy": "true",
                 "mypy-checks": "['mypy-providers']",
             },

docs/apache-airflow/extra-packages-ref.rst

@@ -321,6 +321,8 @@ pre-installed when Airflow is installed.
 +---------------------+-----------------------------------------------------+--------------------------------------+--------------+
 | common-io           | ``pip install 'apache-airflow[common-io]'``         | Core IO Operators                    |              |
 +---------------------+-----------------------------------------------------+--------------------------------------+--------------+
+| common-messaging    | ``pip install 'apache-airflow[common-messaging]'``  | Core Messaging Operators             |              |
++---------------------+-----------------------------------------------------+--------------------------------------+--------------+
 | common-sql          | ``pip install 'apache-airflow[common-sql]'``        | Core SQL Operators                   |      *       |
 +---------------------+-----------------------------------------------------+--------------------------------------+--------------+
 | ftp                 | ``pip install 'apache-airflow[ftp]'``               | FTP hooks and operators              |      *       |

docs/conf.py

@@ -229,6 +229,8 @@
         "_api/airflow/providers/atlassian/index.rst",
         "_api/airflow/providers/cncf/index.rst",
         "_api/airflow/providers/common/index.rst",
+        "_api/airflow/providers/common/messaging/providers/base_provider/index.rst",
+        "_api/airflow/providers/common/messaging/providers/sqs/index.rst",
         "_api/airflow/providers/dbt/index.rst",
         "_api/airflow/providers/microsoft/index.rst",
         *[f"_api/tests/system/{subpackage}/index.rst" for subpackage in empty_subpackages],

generated/provider_dependencies.json

@@ -416,6 +416,18 @@
     "excluded-python-versions": [],
     "state": "ready"
   },
+  "common.messaging": {
+    "deps": [
+      "apache-airflow>=3.0.0.dev0"
+    ],
+    "devel-deps": [],
+    "plugins": [],
+    "cross-providers-deps": [
+      "amazon"
+    ],
+    "excluded-python-versions": [],
+    "state": "ready"
+  },
   "common.sql": {
     "deps": [
       "apache-airflow>=2.9.0",

providers/amazon/src/airflow/providers/amazon/aws/triggers/sqs.py

@@ -23,14 +23,22 @@
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.sqs import SqsHook
 from airflow.providers.amazon.aws.utils.sqs import process_response
-from airflow.triggers.base import BaseTrigger, TriggerEvent
+from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
+
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.triggers.base import BaseEventTrigger, TriggerEvent
+else:
+    from airflow.triggers.base import (  # type: ignore
+        BaseTrigger as BaseEventTrigger,
+        TriggerEvent,
+    )
 
 if TYPE_CHECKING:
     from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection
     from airflow.providers.amazon.aws.utils.sqs import MessageFilteringType
 
 
-class SqsSensorTrigger(BaseTrigger):
+class SqsSensorTrigger(BaseEventTrigger):
     """
     Asynchronously get messages from an Amazon SQS queue and then delete the messages from the queue.
 

providers/common/messaging/README.rst

@@ -0,0 +1,79 @@
+
+.. Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+
+.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+   ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY
+
+Package ``apache-airflow-providers-common-messaging``
+
+Release: ``1.0.0``
+
+
+``Common Messaging Provider``
+
+
+Provider package
+----------------
+
+This is a provider package for ``common.messaging`` provider. All classes for this provider package
+are in ``airflow.providers.common.messaging`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-messaging/1.0.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-common-messaging``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+==================  ==================
+PIP package         Version required
+==================  ==================
+``apache-airflow``  ``>=3.0.0.dev0``
+==================  ==================
+
+Cross provider package dependencies
+-----------------------------------
+
+Those are dependencies that might be needed in order to use all the features of the package.
+You need to install the specified provider packages in order to use them.
+
+You can install such cross-provider dependencies when installing from PyPI. For example:
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-common-messaging[amazon]
+
+
+====================================================================================================  ==========
+Dependent package                                                                                     Extra
+====================================================================================================  ==========
+`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_  ``amazon``
+====================================================================================================  ==========
+
+The changelog for the provider package can be found in the
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-messaging/1.0.0/changelog.html>`_.

providers/common/messaging/docs/changelog.rst

@@ -0,0 +1,35 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. NOTE TO CONTRIBUTORS:
+    Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes
+    and you want to add an explanation to the users on how they are supposed to deal with them.
+    The changelog is updated and maintained semi-automatically by release manager.
+
+``apache-airflow-providers-common-messaging``
+
+Changelog
+---------
+
+1.0.0
+.....
+
+.. note::
+  This release of provider is only available for Airflow 3.0+ as explained in the
+  `Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
+
+Initial version of the provider

providers/common/messaging/docs/commits.rst

@@ -0,0 +1,27 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. THIS FILE IS UPDATED AUTOMATICALLY_AT_RELEASE_TIME
+
+Package apache-airflow-providers-common-messaging
+------------------------------------------------------
+
+``Common Messaging Provider``
+
+
+This is detailed commit list of changes for versions provider package: ``common.messaging``.
+For high-level changelog, see :doc:`package information including changelog <index>`.