diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index bc7e9c70f1d32..401b383ab3bc9 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -64,8 +64,6 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | drill | Integration required for drill operator and hook. | +--------------+-------------------------------------------------------+ -| gremlin | Integration required for gremlin operator and hook. | -+--------------+-------------------------------------------------------+ | kafka | Integration required for Kafka hooks. | +--------------+-------------------------------------------------------+ | kerberos | Integration that provides Kerberos authentication. | @@ -88,6 +86,8 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | statsd | Integration required for Statsd hooks. | +--------------+-------------------------------------------------------+ +| tinkerpop | Integration required for gremlin operator and hook. | ++--------------+-------------------------------------------------------+ | trino | Integration required for Trino hooks. | +--------------+-------------------------------------------------------+ | ydb | Integration required for YDB tests. | diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index c9b04eeef04c8..56ab70d67d0d7 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -332,9 +332,9 @@ --integrationCore Integrations to enable when running (can be more   than one).                                              (all | all-testable | cassandra | celery | drill |      -gremlin | kafka | kerberos | keycloak | mongo | mssql | +kafka | kerberos | keycloak | mongo | mssql |           openlineage | otel | pinot | qdrant | redis | statsd |  -trino | ydb)                                            +tinkerpop | trino | ydb)                                --standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow          r(required for Airflow 3).                               [default: standalone-dag-processor]                     diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 137622302dbdd..4880b90e418bb 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -586,9 +586,9 @@ --integrationCore Integrations to enable when running (can be more   than one).                                              (all | all-testable | cassandra | celery | drill |      -gremlin | kafka | kerberos | keycloak | mongo | mssql | +kafka | kerberos | keycloak | mongo | mssql |           openlineage | otel | pinot | qdrant | redis | statsd |  -trino | ydb)                                            +tinkerpop | trino | ydb)                                --load-example-dags-eEnable configuration to load example DAGs when starting Airflow.                                                --load-default-connections-cEnable configuration to load default connections when   diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index 5aa49fe26cad0..62145fd5a6574 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -d05dcea3dd1b9922f1d0017959f8735d +cc761daf3b1f5d3801aebdb122d09ac4 diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 4a47f57411b57..1fa6082d34bf2 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -497,9 +497,9 @@ --integrationCore Integrations to enable when running (can be more   than one).                                              (all | all-testable | cassandra | celery | drill |      -gremlin | kafka | kerberos | keycloak | mongo | mssql | +kafka | kerberos | keycloak | mongo | mssql |           openlineage | otel | pinot | qdrant | redis | statsd |  -trino | ydb)                                            +tinkerpop | trino | ydb)                                --standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow          r(required for Airflow 3).                               [default: standalone-dag-processor]                     diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 461967f90a842..29576c21a5861 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -6f7f219bac0ca1955590f8f1baf213ef +9d93476551743464bdb45703cefc78ef diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg index e4bdee626d497..b85c795baae68 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg @@ -237,8 +237,8 @@ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Integration tests ──────────────────────────────────────────────────────────────────────────────────────────────────╮ --integrationProviders Integration(s) to enable when running (can be more than one).                             -(all | all-testable | cassandra | celery | drill | gremlin | kafka | mongo | mssql | openlineage |  -pinot | qdrant | redis | trino | ydb)                                                               +(all | all-testable | cassandra | celery | drill | kafka | mongo | mssql | openlineage | pinot |    +qdrant | redis | tinkerpop | trino | ydb)                                                           ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ --github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow] diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt index e2e41d2494d93..df39e41d7446c 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt @@ -1 +1 @@ -d03228589829620ec13e6b7a2fce1901 +d62ead0f3d78554ef853e4311c27c572 diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 7f88b8bffaf44..3efcb54353b21 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -216,6 +216,7 @@ def _run_test( parallel_test_types_list=shell_params.parallel_test_types_list, keep_env_variables=shell_params.keep_env_variables, no_db_cleanup=shell_params.no_db_cleanup, + integration=shell_params.integration, ) pytest_args.extend(extra_pytest_args) # Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index b16293f1318ab..736d64dc58a8f 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -68,7 +68,7 @@ "celery", "cassandra", "drill", - "gremlin", + "tinkerpop", "kafka", "mongo", "mssql", diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 74cc0c6281975..62f9463257c43 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -245,14 +245,26 @@ def convert_test_type_to_pytest_args( *, test_group: GroupOfTests, test_type: str, + integration: tuple | None = None, ) -> list[str]: if test_type == "None": return [] if test_type in ALL_TEST_SUITES: - return [ + all_paths = [ *TEST_GROUP_TO_TEST_FOLDERS[test_group], *ALL_TEST_SUITES[test_type], ] + + if integration and test_group == GroupOfTests.INTEGRATION_PROVIDERS: + filtered_paths = [ + path + for path in all_paths + if any(path.endswith(f"{value}/tests/integration") for value in integration) + ] + + return filtered_paths + return all_paths + if test_group == GroupOfTests.SYSTEM and test_type != NONE_TEST_TYPE: get_console().print(f"[error]Only {NONE_TEST_TYPE} should be allowed as test type[/]") sys.exit(1) @@ -336,6 +348,7 @@ def generate_args_for_pytest( python_version: str, keep_env_variables: bool, no_db_cleanup: bool, + integration: tuple | None = None, ): result_log_file, warnings_file, coverage_file = test_paths(test_type, backend) if skip_db_tests and parallel_test_types_list: @@ -347,6 +360,7 @@ def generate_args_for_pytest( args = convert_test_type_to_pytest_args( test_group=test_group, test_type=test_type, + integration=integration, ) args.extend( [ diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index b249838cf241b..fa2d3b53070ba 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1153,7 +1153,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "testable-core-integrations": "['kerberos']", - "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'gremlin', 'kafka', " + "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'tinkerpop', 'kafka', " "'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']", "needs-mypy": "true", "mypy-checks": ALL_MYPY_CHECKS, diff --git a/scripts/ci/docker-compose/integration-gremlin.yml b/scripts/ci/docker-compose/integration-tinkerpop.yml similarity index 100% rename from scripts/ci/docker-compose/integration-gremlin.yml rename to scripts/ci/docker-compose/integration-tinkerpop.yml