Remove pytest.mark.db_test: airbyte and amazon providers where possible

.pre-commit-config.yaml

@@ -570,6 +570,16 @@ repos:
         entry: "pydevd.*settrace\\("
         pass_filenames: true
         files: \.py$
+      - id: check-pytest-mark-db-test-in-providers
+        language: pygrep
+        name: Check pytest.mark.db_test use in providers
+        entry: pytest\.mark\.db_test
+        pass_filenames: true
+        # Here we should add providers that are already free from the pytest.mark.db_test
+        # and we want to keep them clean and only use non-db-tests
+        files: >
+            (?x)
+            ^providers/airbyte/.*\.py$
       - id: check-links-to-example-dags-do-not-use-hardcoded-versions
         name: Verify no hard-coded version in example dags
         description: The links to example dags should use |version| as version specification

contributing-docs/08_static_code_checks.rst

@@ -221,6 +221,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-pydevd-left-in-code                                 | Check for pydevd debug statements accidentally left    |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| check-pytest-mark-db-test-in-providers                    | Check pytest.mark.db_test use in providers             |         |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-revision-heads-map                                  | Check that the REVISION_HEADS_MAP is up-to-date        |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-safe-filter-usage-in-html                           | Don't use safe in templates                            |         |

dev/breeze/doc/images/output_static-checks.txt

@@ -1 +1 @@
-f31561bb0408a8cab278aad420d260b3
+bb6a22bca8cde603acddfad0f073b3c0

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -74,6 +74,7 @@
     "check-provider-yaml-valid",
     "check-providers-subpackages-init-file-exist",
     "check-pydevd-left-in-code",
+    "check-pytest-mark-db-test-in-providers",
     "check-revision-heads-map",
     "check-safe-filter-usage-in-html",
     "check-significant-newsfragments-are-valid",

providers/airbyte/tests/unit/airbyte/hooks/test_airbyte.py

@@ -27,11 +27,7 @@
 from airflow.models import Connection
 from airflow.providers.airbyte.hooks.airbyte import AirbyteHook
 
-# those tests will not work with database isolation because they mock requests
-pytestmark = pytest.mark.db_test
 
-
-@pytest.mark.db_test
 class TestAirbyteHook:
     """
     Test all functions from Airbyte Hook

providers/airbyte/tests/unit/airbyte/operators/test_airbyte.py

@@ -19,14 +19,12 @@
 
 from unittest import mock
 
-import pytest
 from airbyte_api.models import JobCreateRequest, JobResponse, JobStatusEnum, JobTypeEnum
 
 from airflow.models import Connection
 from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator
 
 
-@pytest.mark.db_test
 class TestAirbyteTriggerSyncOp:
     """
     Test execute function from Airbyte Operator

providers/airbyte/tests/unit/airbyte/sensors/test_airbyte.py

@@ -27,7 +27,6 @@
 from airflow.providers.airbyte.sensors.airbyte import AirbyteJobSensor
 
 
-@pytest.mark.db_test
 class TestAirbyteJobSensor:
     task_id = "task-id"
     airbyte_conn_id = "airbyte-conn-test"

providers/airbyte/tests/unit/airbyte/triggers/test_airbyte.py

@@ -29,7 +29,6 @@
 from airflow.triggers.base import TriggerEvent
 
 
-@pytest.mark.db_test
 class TestAirbyteSyncTrigger:
     DAG_ID = "airbyte_sync_run"
     TASK_ID = "airbyte_sync_run_task_op"

providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

@@ -61,8 +61,6 @@
 from tests_common.test_utils.config import conf_vars
 from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
 
-pytestmark = pytest.mark.db_test
-
 airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3]))
 
 ARN1 = "arn1"

providers/amazon/tests/unit/amazon/aws/hooks/test_base_aws.py

@@ -883,7 +883,6 @@ def test_hook_connection_test(self):
         assert result
         assert hook.client_type == "s3"  # Same client_type which defined during initialisation
 
-    @pytest.mark.db_test
     @mock.patch("boto3.session.Session")
     def test_hook_connection_test_failed(self, mock_boto3_session):
         """Test ``test_connection`` failure."""
@@ -1135,7 +1134,6 @@ def test_waiter_config_params_not_provided(waiter_path_mock: MagicMock, caplog):
     assert "PARAM_1" in str(ae.value)
 
 
-@pytest.mark.db_test
 @mock.patch.object(AwsGenericHook, "waiter_path", new_callable=PropertyMock)
 def test_waiter_config_no_params_needed(waiter_path_mock: MagicMock, caplog):
     waiter_path_mock.return_value = TEST_WAITER_CONFIG_LOCATION

providers/amazon/tests/unit/amazon/aws/hooks/test_chime.py

@@ -25,8 +25,6 @@
 from airflow.models import Connection
 from airflow.providers.amazon.aws.hooks.chime import ChimeWebhookHook
 
-pytestmark = pytest.mark.db_test
-
 
 class TestChimeWebhookHook:
     _config = {

providers/amazon/tests/unit/amazon/aws/hooks/test_emr.py

@@ -154,7 +154,6 @@ def test_add_job_flow_steps_raises_exception_on_failure(self, mock_conn, mock_sl
         assert "test failure details" in caplog.messages[-1]
         mock_conn.get_waiter.assert_called_with("step_complete")
 
-    @pytest.mark.db_test
     @mock_aws
     def test_create_job_flow_extra_args(self):
         """

providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py

@@ -1272,7 +1272,6 @@ def test_delete_bucket_if_bucket_not_exist(self, s3_bucket):
             assert mock_hook.delete_bucket(bucket_name="not-exists-bucket-name", force_delete=True)
         assert ctx.value.response["Error"]["Code"] == "NoSuchBucket"
 
-    @pytest.mark.db_test
     def test_provide_bucket_name(self):
         with mock.patch.object(
             S3Hook,
@@ -1728,7 +1727,6 @@ def test_delete_bucket_tagging_with_no_tags(self):
             hook.get_bucket_tagging(bucket_name="new_bucket")
 
 
-@pytest.mark.db_test
 @pytest.mark.parametrize(
     "key_kind, has_conn, has_bucket, precedence, expected",
     [

providers/amazon/tests/unit/amazon/aws/hooks/test_sagemaker_unified_studio.py

@@ -28,8 +28,6 @@
 )
 from airflow.utils.session import create_session
 
-pytestmark = pytest.mark.db_test
-
 
 class TestSageMakerNotebookHook:
     @pytest.fixture(autouse=True)
@@ -174,6 +172,7 @@ def test_handle_unexpected_state(self):
         with pytest.raises(AirflowException, match=error_message):
             self.hook._handle_state(execution_id, status, error_message)
 
+    @pytest.mark.db_test
     @patch(
         "airflow.providers.amazon.aws.hooks.sagemaker_unified_studio.SageMakerNotebookHook._set_xcom_files"
     )
@@ -187,6 +186,7 @@ def test_set_xcom_files_negative_missing_context(self):
         with pytest.raises(AirflowException, match="context is required"):
             self.hook._set_xcom_files(self.files, {})
 
+    @pytest.mark.db_test
     @patch(
         "airflow.providers.amazon.aws.hooks.sagemaker_unified_studio.SageMakerNotebookHook._set_xcom_s3_path"
     )

providers/amazon/tests/unit/amazon/aws/links/test_athena.py

@@ -16,6 +16,8 @@
 # under the License.
 from __future__ import annotations
 
+import pytest
+
 from airflow.providers.amazon.aws.links.athena import AthenaQueryResultsLink
 from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
 
@@ -24,6 +26,8 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+pytestmark = pytest.mark.db_test
+
 
 class TestAthenaQueryResultsLink(BaseAwsLinksTestCase):
     link_class = AthenaQueryResultsLink

providers/amazon/tests/unit/amazon/aws/links/test_base_aws.py

@@ -132,7 +132,6 @@ class OperatorAndTi(NamedTuple):
     task_instance: TaskInstance
 
 
-@pytest.mark.db_test
 @pytest.mark.need_serialized_dag
 class BaseAwsLinksTestCase:
     """Base class for AWS Provider links tests."""

providers/amazon/tests/unit/amazon/aws/links/test_batch.py

@@ -16,6 +16,8 @@
 # under the License.
 from __future__ import annotations
 
+import pytest
+
 from airflow.providers.amazon.aws.links.batch import (
     BatchJobDefinitionLink,
     BatchJobDetailsLink,
@@ -28,6 +30,8 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+pytestmark = pytest.mark.db_test
+
 
 class TestBatchJobDefinitionLink(BaseAwsLinksTestCase):
     link_class = BatchJobDefinitionLink

providers/amazon/tests/unit/amazon/aws/links/test_comprehend.py

@@ -27,6 +27,10 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
 
 class TestComprehendPiiEntitiesDetectionLink(BaseAwsLinksTestCase):
     link_class = ComprehendPiiEntitiesDetectionLink

providers/amazon/tests/unit/amazon/aws/links/test_datasync.py

@@ -24,6 +24,11 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
+
 TASK_ID = "task-0b36221bf94ad2bdd"
 EXECUTION_ID = "exec-00000000000000004"
 

providers/amazon/tests/unit/amazon/aws/links/test_ec2.py

@@ -24,6 +24,10 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
 
 class TestEC2InstanceLink(BaseAwsLinksTestCase):
     link_class = EC2InstanceLink

providers/amazon/tests/unit/amazon/aws/links/test_emr.py

@@ -36,6 +36,9 @@
 
 from unit.amazon.aws.links.test_base_aws import BaseAwsLinksTestCase
 
+pytestmark = pytest.mark.db_test
+
+
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 

providers/amazon/tests/unit/amazon/aws/links/test_glue.py

@@ -24,6 +24,10 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
 
 class TestGlueJobRunDetailsLink(BaseAwsLinksTestCase):
     link_class = GlueJobRunDetailsLink

providers/amazon/tests/unit/amazon/aws/links/test_logs.py

@@ -24,6 +24,10 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
 
 class TestCloudWatchEventsLink(BaseAwsLinksTestCase):
     link_class = CloudWatchEventsLink

providers/amazon/tests/unit/amazon/aws/links/test_sagemaker.py

@@ -25,6 +25,10 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
 
 class TestSageMakerTransformDetailsLink(BaseAwsLinksTestCase):
     link_class = SageMakerTransformJobLink

providers/amazon/tests/unit/amazon/aws/links/test_sagemaker_unified_studio.py

@@ -24,6 +24,10 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+import pytest
+
+pytestmark = pytest.mark.db_test
+
 
 class TestSageMakerUnifiedStudioLink(BaseAwsLinksTestCase):
     link_class = SageMakerUnifiedStudioLink

providers/amazon/tests/unit/amazon/aws/links/test_step_function.py

@@ -29,6 +29,8 @@
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk.execution_time.comms import XComResult
 
+pytestmark = pytest.mark.db_test
+
 
 class TestStateMachineDetailsLink(BaseAwsLinksTestCase):
     link_class = StateMachineDetailsLink

providers/amazon/tests/unit/amazon/aws/sensors/test_dynamodb.py

@@ -17,14 +17,11 @@
 
 from __future__ import annotations
 
-import pytest
 from moto import mock_aws
 
 from airflow.providers.amazon.aws.hooks.dynamodb import DynamoDBHook
 from airflow.providers.amazon.aws.sensors.dynamodb import DynamoDBValueSensor
 
-pytestmark = pytest.mark.db_test
-
 
 class TestDynamoDBValueSensor:
     def setup_method(self):

providers/amazon/tests/unit/amazon/aws/transfers/test_google_api_to_s3.py

@@ -26,7 +26,6 @@
 from airflow.providers.amazon.aws.transfers.google_api_to_s3 import GoogleApiToS3Operator
 
 # This test mocks json.dumps so it won't work for database isolation mode
-pytestmark = pytest.mark.db_test
 
 
 class TestGoogleApiToS3:

providers/amazon/tests/unit/amazon/aws/triggers/test_serialization.py

@@ -64,8 +64,6 @@
 from airflow.providers.amazon.aws.utils.rds import RdsDbType
 from airflow.serialization.serialized_objects import BaseSerialization
 
-pytestmark = pytest.mark.db_test
-
 BATCH_JOB_ID = "job_id"
 
 TEST_CLUSTER_IDENTIFIER = "test-cluster"

providers/amazon/tests/unit/amazon/aws/utils/test_suppress.py

@@ -22,7 +22,6 @@
 from airflow.providers.amazon.aws.utils.suppress import return_on_error
 
 
-@pytest.mark.db_test
 def test_suppress_function(caplog):
     @return_on_error("error")
     def fn(value: str, exc: Exception | None = None) -> str:

providers/amazon/tests/unit/amazon/aws/waiters/test_eks.py

@@ -19,6 +19,7 @@
 import json
 
 import boto3
+import pytest
 from moto import mock_aws
 
 from airflow.providers.amazon.aws.hooks.eks import EksHook
@@ -36,6 +37,7 @@ def test_service_waiters(self):
             assert waiter in hook.list_waiters()
             assert waiter in hook._list_custom_waiters()
 
+    @pytest.mark.db_test
     @mock_aws
     def test_existing_waiter_inherited(self):
         """