From a4fdf72b800c075c31697c0d54027d1be86c26ec Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 23 Jul 2023 08:53:31 +0200 Subject: [PATCH] Quarantine test_backfill_integration in dask executor The test has been recently failing with deadlock (see #32778) and needs thorough looking at if we want to find the root cause/remedium. In the meantime it looks like a niche case connected with Dask Executor that is rather obsure and we have no expertise in solving problems with and diagnosing, so until the problem is diagnosed it might be a long time (and maybe even we decide not to care about it and let Dask community take a look and either fix or ignore it. We aim to have a very low number of those Quarantined tests (currently we have 1 and we have not run it for a while as this was a mysql test run on Postgres) but we have now the opportunity to also improve the quarantined tests framework. This test will be run together with other (1) quarantined test and: * they will not be run in our regular tests * they are run sequentially not in parallel with all other tests * they are run for all 4 backends but only for the default versions of those backends * failure of the quarantined tests will not cause failure of the whole job or limit constraints from being generated and updated --- .github/workflows/ci.yml | 47 +++++++- .../commands/testing_commands.py | 2 + images/breeze/output-commands-hash.txt | 4 +- images/breeze/output_testing.svg | 24 ++-- images/breeze/output_testing_tests.svg | 114 +++++++++--------- tests/executors/test_dask_executor.py | 4 + 6 files changed, 118 insertions(+), 77 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc0c1838252ba..cda3148c59a10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1308,15 +1308,13 @@ jobs: needs: [build-info, wait-for-ci-images] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "Quarantined" + TEST_TYPE: "Quarantined" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" - BACKEND: "sqlite" - BACKEND_VERSION: "" JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" COVERAGE: "${{needs.build-info.outputs.run-coverage}}" - if: needs.build-info.outputs.run-tests == 'true' + if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.runs-on == 'self-hosted' steps: - name: Cleanup repo shell: bash @@ -1329,8 +1327,45 @@ jobs: Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image - name: > - Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined - run: breeze testing tests --run-in-parallel || true + Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "postgres" + BACKEND_VERSION: "11" + POSTGRES_VERSION: "11" + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down + - name: > + Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "mysql" + BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}} + MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}} + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down + - name: > + Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "mssql" + BACKEND_VERSION: ${{needs.build-info.outputs.default-mssql-version}} + MSSQL_VERSION: ${{needs.build-info.outputs.default-mssql-version}} + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down + - name: > + Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "sqlite" + BACKEND_VERSION: "" + MSSQL_VERSION: "" + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down - name: > Post Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined" uses: ./.github/actions/post_tests diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 12e7ad98b02a3..90109b90f4814 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -331,12 +331,14 @@ def run_tests_in_parallel( "should be run: `Providers[airbyte,http]` or " "excluded from the full test suite: `Providers[-amazon,google]`", default="All", + envvar="TEST_TYPE", type=NotVerifiedBetterChoice(ALLOWED_TEST_TYPE_CHOICES), ) @click.option( "--test-timeout", help="Test timeout. Set the pytest setup, execution and teardown timeouts to this value", default=60, + envvar="TEST_TIMEOUT", type=IntRange(min=0), show_default=True, ) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index a678af7996c67..b3fc1ec2265fc 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -66,5 +66,5 @@ static-checks:f9ec0d7edaba84180403d95469d94ea0 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3 -testing:tests:e18fbd845ca5783879244bf0f9f9c51e -testing:f360a66d2b1b4c39880117ae6166007f +testing:tests:3c202e65824e405269e78f58936980e0 +testing:68a089bc30e0e60f834f205df1e9f086 diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index 82c6f8e344e3c..7d14280b52730 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-r5 { fill: #868887 } +.breeze-testing-r4 { fill: #868887 } +.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -96,19 +96,19 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│integration-tests                        Run the specified integration tests.                                     │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│integration-tests                        Run the specified integration tests.                                     │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index ec3da499fa7f6..9cdc350da969f 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -35,8 +35,8 @@ .breeze-testing-tests-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-tests-r2 { fill: #c5c8c6 } .breeze-testing-tests-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-tests-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-tests-r5 { fill: #868887 } +.breeze-testing-tests-r4 { fill: #868887 } +.breeze-testing-tests-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-tests-r6 { fill: #98a84b;font-weight: bold } .breeze-testing-tests-r7 { fill: #8d7b39 } @@ -232,64 +232,64 @@ -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit test targets. -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--test-typeType of test to run. With Providers, you can specify tests of which providers should be    │ -│run: `Providers[airbyte,http]` or excluded from the full test suite:                       │ -│`Providers[-amazon,google]`                                                                │ -│(All | API | Always | CLI | Core | Other | Providers | WWW | PlainAsserts | Postgres |     │ -│MySQL | Quarantine)                                                                        │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--collect-onlyCollect tests only, do not run them.│ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot | statsd│ -│| statsd | trino)                                                                          │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--parallel-test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Other Providers WWW PlainAsserts] │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, │ -│or `sdist` if Airflow should be removed, installed from wheel packages or sdist     │ -│packages available in dist folder respectively. Implies --mount-sources `remove`.   │ -│(none | wheel | sdist | <airflow_version>)                                          │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =│ -│selected).                                                                          │ -│(selected | all | skip | remove)                                                    │ -│[default: selected]                                                                 │ -│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ -│--remove-arm-packagesRemoves arm packages from the image to test if ARM collection works│ -│--skip-docker-compose-downSkips running docker-compose down after tests│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--test-typeType of test to run. With Providers, you can specify tests of which providers should be    │ +│run: `Providers[airbyte,http]` or excluded from the full test suite:                       │ +│`Providers[-amazon,google]`                                                                │ +│(All | API | Always | CLI | Core | Other | Providers | WWW | PlainAsserts | Postgres |     │ +│MySQL | Quarantine)                                                                        │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--collect-onlyCollect tests only, do not run them.│ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot | statsd│ +│| statsd | trino)                                                                          │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--parallel-test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Other Providers WWW PlainAsserts] │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, │ +│or `sdist` if Airflow should be removed, installed from wheel packages or sdist     │ +│packages available in dist folder respectively. Implies --mount-sources `remove`.   │ +│(none | wheel | sdist | <airflow_version>)                                          │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =│ +│selected).                                                                          │ +│(selected | all | skip | remove)                                                    │ +│[default: selected]                                                                 │ +│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ +│--remove-arm-packagesRemoves arm packages from the image to test if ARM collection works│ +│--skip-docker-compose-downSkips running docker-compose down after tests│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/tests/executors/test_dask_executor.py b/tests/executors/test_dask_executor.py index f64f2a7dfdc72..d8ded41e158bb 100644 --- a/tests/executors/test_dask_executor.py +++ b/tests/executors/test_dask_executor.py @@ -101,6 +101,10 @@ def test_dask_executor_functions(self): executor = DaskExecutor(cluster_address=self.cluster.scheduler_address) self.assert_tasks_on_executor(executor, timeout_executor=120) + # This test is quarantined because it became rather flaky on our CI in July 2023 and reason for this + # is unknown. An issue for that was created: https://github.com/apache/airflow/issues/32778 and the + # marker should be removed while (possibly) the reason for flaky behaviour is found and fixed. + @pytest.mark.quarantined @pytest.mark.execution_timeout(180) def test_backfill_integration(self): """