diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 49ed1fc276114..7be646c52a7e5 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -116,7 +116,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} - VERSION_SUFFIX_FOR_PYPI: "dev0" GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6af4796ccb1f1..a18ce06b566b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -466,26 +466,26 @@ jobs: debug-resources: ${{ needs.build-info.outputs.debug-resources }} use-uv: ${{ needs.build-info.outputs.use-uv }} - tests-integration-system: - name: Integration and System Tests - needs: [build-info, build-ci-images] - uses: ./.github/workflows/integration-system-tests.yml - permissions: - contents: read - packages: read - with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} - testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} - run-system-tests: ${{ needs.build-info.outputs.run-tests }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} - default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} - default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - if: needs.build-info.outputs.run-tests == 'true' + # tests-integration-system: + # name: Integration and System Tests + # needs: [build-info, build-ci-images] + # uses: ./.github/workflows/integration-system-tests.yml + # permissions: + # contents: read + # packages: read + # with: + # runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + # testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} + # testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} + # run-system-tests: ${{ needs.build-info.outputs.run-tests }} + # default-python-version: ${{ needs.build-info.outputs.default-python-version }} + # default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} + # default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }} + # skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + # run-coverage: ${{ needs.build-info.outputs.run-coverage }} + # debug-resources: ${{ needs.build-info.outputs.debug-resources }} + # use-uv: ${{ needs.build-info.outputs.use-uv }} + # if: needs.build-info.outputs.run-tests == 'true' tests-with-lowest-direct-resolution: name: "Lowest direct dependency providers tests" @@ -593,7 +593,6 @@ jobs: - tests-mysql - tests-postgres - tests-non-db - - tests-integration-system - build-prod-images uses: ./.github/workflows/finalize-tests.yml with: diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1b4aa19cbe595..2b72e08be7d4c 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -117,7 +117,7 @@ jobs: - name: "Helm release tarball" run: > breeze release-management prepare-helm-chart-tarball --ignore-version-check --override-tag - --skip-tag-signing --version 0.0.0 --version-suffix dev0 + --skip-tag-signing --version 0.0.0 - name: Generate GPG key for signing # Sometimes the key will be already added to the keyring, so we ignore the error run: gpg --batch --passphrase '' --quick-gen-key dev@airflow.apache.org default default || true diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 96e653d10ca19..4a662eb460753 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -126,7 +126,6 @@ jobs: if: inputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - VERSION_SUFFIX_FOR_PYPI: ${{ inputs.branch == 'main' && 'dev0' || '' }} steps: - name: "Cleanup repo" shell: bash @@ -197,7 +196,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} - VERSION_SUFFIX_FOR_PYPI: ${{ inputs.branch == 'main' && 'dev0' || '' }} INCLUDE_NOT_READY_PROVIDERS: "true" # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the # Airflow one is going to be used diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 60f3be1b181e4..edaa32e60bcef 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -116,7 +116,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" UPGRADE_TO_NEWER_DEPENDENCIES: "false" VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: "Cleanup repo" shell: bash @@ -191,7 +190,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" UPGRADE_TO_NEWER_DEPENDENCIES: "false" VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" if: inputs.include-prod-images == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 5fe9eaad7f249..20901d88cb5d7 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -145,16 +145,13 @@ jobs: - name: "Create airflow_cache builder" run: docker buildx create --name airflow_cache - name: "Prepare chicken-eggs provider packages" - # In case of provider packages which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 shell: bash env: CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} run: > breeze release-management prepare-provider-packages --package-format wheel - --version-suffix-for-pypi dev0 ${CHICKEN_EGG_PROVIDERS} + ${CHICKEN_EGG_PROVIDERS} if: needs.build-info.outputs.chicken-egg-providers != '' - name: "Copy dist packages to docker-context files" shell: bash diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 09e08b7561c9a..39566a6042c7c 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -200,27 +200,3 @@ jobs: debug-resources: ${{ inputs.debug-resources }} use-uv: ${{ inputs.use-uv }} if: ${{ inputs.default-branch == 'main' }} - - # matrix.test-group comes from run-unit-tests.yml - tests-system: - name: "System test: ${{ matrix.test-group }}" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - test-name: "SystemTest" - test-scope: "System" - test-groups: ${{ inputs.test-groups }} - backend: "postgres" - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} - excludes: "[]" - core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} - providers-test-types-list-as-string: ${{ inputs.providers-test-types-list-as-string }} - include-success-outputs: ${{ inputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/test-provider-packages.yml b/.github/workflows/test-provider-packages.yml index 4aa2f4116e35c..a008f4290f705 100644 --- a/.github/workflows/test-provider-packages.yml +++ b/.github/workflows/test-provider-packages.yml @@ -99,15 +99,15 @@ jobs: - name: "Prepare provider documentation" run: > breeze release-management prepare-provider-documentation --include-not-ready-providers - --non-interactive + --non-interactive fab if: matrix.package-format == 'wheel' - name: "Prepare provider packages: ${{ matrix.package-format }}" run: > breeze release-management prepare-provider-packages --include-not-ready-providers - --version-suffix-for-pypi dev0 --package-format ${{ matrix.package-format }} + --package-format ${{ matrix.package-format }} fab common.compat - name: "Prepare airflow package: ${{ matrix.package-format }}" run: > - breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 + breeze release-management prepare-airflow-package --package-format ${{ matrix.package-format }} - name: "Verify ${{ matrix.package-format }} packages with twine" run: | @@ -167,7 +167,6 @@ jobs: GITHUB_USERNAME: ${{ github.actor }} INCLUDE_NOT_READY_PROVIDERS: "true" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" - VERSION_SUFFIX_FOR_PYPI: "dev0" VERBOSE: "true" CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}" if: inputs.skip-providers-tests != 'true' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3bea00cebeb5..259408d1ae675 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1111,7 +1111,7 @@ repos: stages: ['manual'] name: Run mypy for providers (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py providers/src/airflow/providers + entry: ./scripts/ci/pre_commit/mypy_folder.py airflow/providers/fab pass_filenames: false files: ^.*\.py$ require_serial: true diff --git a/Dockerfile.ci b/Dockerfile.ci index 2004160a6f34d..04cbe45928fe4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1041,7 +1041,7 @@ function check_force_lowest_dependencies() { EXTRA="[devel]" if [[ ${TEST_TYPE=} =~ Providers\[.*\] ]]; then # shellcheck disable=SC2001 - EXTRA=$(echo "[${TEST_TYPE},devel]" | sed 's/Providers\[\(.*\)\]/\1/') + EXTRA=$(echo "[${TEST_TYPE}]" | sed 's/Providers\[\([^]]*\)\]/\1,devel/') echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${EXTRA}${COLOR_RESET}" echo diff --git a/airflow/api_connexion/security.py b/airflow/api_connexion/security.py index 660bc6cce2370..c68094a521245 100644 --- a/airflow/api_connexion/security.py +++ b/airflow/api_connexion/security.py @@ -45,13 +45,14 @@ def check_authentication() -> None: """Check that the request has valid authorization information.""" - for auth in get_airflow_app().api_auth: + airflow_app = get_airflow_app() + for auth in airflow_app.api_auth: response = auth.requires_authentication(Response)() if response.status_code == 200: return # Even if the current_user is anonymous, the AUTH_ROLE_PUBLIC might still have permission. - appbuilder = get_airflow_app().appbuilder + appbuilder = airflow_app.appbuilder if appbuilder.get_app.config.get("AUTH_ROLE_PUBLIC", None): return diff --git a/airflow/providers/common/compat/openlineage/utils/utils.py b/airflow/providers/common/compat/openlineage/utils/utils.py index 5492c76d55dcc..0d393b5126751 100644 --- a/airflow/providers/common/compat/openlineage/utils/utils.py +++ b/airflow/providers/common/compat/openlineage/utils/utils.py @@ -20,24 +20,32 @@ from functools import wraps from typing import TYPE_CHECKING -if TYPE_CHECKING: - from airflow.providers.openlineage.utils.utils import translate_airflow_asset -else: - try: - from airflow.providers.openlineage.utils.utils import translate_airflow_asset - except ImportError: - from airflow.providers.openlineage.utils.utils import translate_airflow_dataset - - def rename_asset_as_dataset(function): - @wraps(function) - def wrapper(*args, **kwargs): - if "asset" in kwargs: - kwargs["dataset"] = kwargs.pop("asset") - return function(*args, **kwargs) - - return wrapper - - translate_airflow_asset = rename_asset_as_dataset(translate_airflow_dataset) - - -__all__ = ["translate_airflow_asset"] +from airflow.exceptions import AirflowOptionalProviderFeatureException + +try: + if TYPE_CHECKING: + try: + from airflow.providers.openlineage.utils.utils import ( # type:ignore [attr-defined] + translate_airflow_asset, + ) + except ImportError: + raise AirflowOptionalProviderFeatureException() + else: + try: + from airflow.providers.openlineage.utils.utils import translate_airflow_asset + except ImportError: + from airflow.providers.openlineage.utils.utils import translate_airflow_dataset + + def rename_asset_as_dataset(function): + @wraps(function) + def wrapper(*args, **kwargs): + if "asset" in kwargs: + kwargs["dataset"] = kwargs.pop("asset") + return function(*args, **kwargs) + + return wrapper + + translate_airflow_asset = rename_asset_as_dataset(translate_airflow_dataset) + __all__ = ["translate_airflow_asset"] +except ImportError: + raise AirflowOptionalProviderFeatureException() diff --git a/airflow/providers/fab/README.rst b/airflow/providers/fab/README.rst new file mode 100644 index 0000000000000..f1e4eac1ac641 --- /dev/null +++ b/airflow/providers/fab/README.rst @@ -0,0 +1,87 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +Package ``apache-airflow-providers-fab`` + +Release: ``1.5.4`` + + +`Flask App Builder `__ + + +Provider package +---------------- + +This is a provider package for ``fab`` provider. All classes for this provider package +are in ``airflow.providers.fab`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-fab`` + +The package supports the following python versions: 3.10,3.11,3.12 + +Requirements +------------ + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=2.9.0`` +``apache-airflow-providers-common-compat`` ``>=1.2.1`` +``flask-login`` ``>=0.6.3`` +``flask-session`` ``>=0.8.0`` +``flask`` ``>=2.2,<3`` +``flask-appbuilder`` ``==4.5.4`` +``google-re2`` ``>=1.0`` +``jmespath`` ``>=0.7.0`` +========================================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-fab[common.compat] + + +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +================================================================================================================== ================= + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/airflow/providers/fab/auth_manager/cli_commands/db_command.py b/airflow/providers/fab/auth_manager/cli_commands/db_command.py index 8b41cf4216c87..861953a0fe2b4 100644 --- a/airflow/providers/fab/auth_manager/cli_commands/db_command.py +++ b/airflow/providers/fab/auth_manager/cli_commands/db_command.py @@ -17,10 +17,19 @@ from __future__ import annotations from airflow import settings -from airflow.cli.commands.db_command import run_db_downgrade_command, run_db_migrate_command -from airflow.providers.fab.auth_manager.models.db import _REVISION_HEADS_MAP, FABDBManager -from airflow.utils import cli as cli_utils -from airflow.utils.providers_configuration_loader import providers_configuration_loaded + +try: + from airflow.cli.commands.db_command import ( # type:ignore [attr-defined] + run_db_downgrade_command, + run_db_migrate_command, + ) + from airflow.providers.fab.auth_manager.models.db import _REVISION_HEADS_MAP, FABDBManager + from airflow.utils import cli as cli_utils + from airflow.utils.providers_configuration_loader import providers_configuration_loaded +except ImportError: + from airflow.exceptions import AirflowOptionalProviderFeatureException + + raise AirflowOptionalProviderFeatureException() @providers_configuration_loaded diff --git a/airflow/providers/fab/auth_manager/cli_commands/utils.py b/airflow/providers/fab/auth_manager/cli_commands/utils.py index e848c2094ce5b..1a3efa84f97c4 100644 --- a/airflow/providers/fab/auth_manager/cli_commands/utils.py +++ b/airflow/providers/fab/auth_manager/cli_commands/utils.py @@ -42,7 +42,7 @@ def _return_appbuilder(app: Flask) -> AirflowAppBuilder: """Return an appbuilder instance for the given app.""" init_appbuilder(app) init_plugins(app) - init_airflow_session_interface(app) + init_airflow_session_interface(app, None) return app.appbuilder # type: ignore[attr-defined] diff --git a/airflow/providers/fab/auth_manager/fab_auth_manager.py b/airflow/providers/fab/auth_manager/fab_auth_manager.py index e93e440f5ddfe..a0f7cdef057fe 100644 --- a/airflow/providers/fab/auth_manager/fab_auth_manager.py +++ b/airflow/providers/fab/auth_manager/fab_auth_manager.py @@ -95,7 +95,7 @@ ) from airflow.providers.common.compat.assets import AssetDetails from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride - from airflow.security.permissions import RESOURCE_ASSET + from airflow.security.permissions import RESOURCE_ASSET # type: ignore[attr-defined] else: from airflow.providers.common.compat.security.permissions import RESOURCE_ASSET @@ -403,9 +403,7 @@ def get_url_logout(self): def get_url_user_profile(self) -> str | None: """Return the url to a page displaying info about the current user.""" - if not self.security_manager.user_view or self.appbuilder.get_app.config.get( - "AUTH_ROLE_PUBLIC", None - ): + if not self.security_manager.user_view: return None return url_for(f"{self.security_manager.user_view.endpoint}.userinfo") diff --git a/airflow/providers/fab/auth_manager/models/db.py b/airflow/providers/fab/auth_manager/models/db.py index ce0efef55a1cd..ae7d39f7216c0 100644 --- a/airflow/providers/fab/auth_manager/models/db.py +++ b/airflow/providers/fab/auth_manager/models/db.py @@ -18,11 +18,14 @@ from pathlib import Path -from airflow import settings -from airflow.exceptions import AirflowException -from airflow.providers.fab.auth_manager.models import metadata -from airflow.utils.db import _offline_migration, print_happy_cat -from airflow.utils.db_manager import BaseDBManager +try: + from airflow import settings + from airflow.exceptions import AirflowException, AirflowOptionalProviderFeatureException + from airflow.providers.fab.auth_manager.models import metadata + from airflow.utils.db import _offline_migration, print_happy_cat + from airflow.utils.db_manager import BaseDBManager +except ImportError: + raise AirflowOptionalProviderFeatureException() PACKAGE_DIR = Path(__file__).parents[2] diff --git a/airflow/providers/fab/auth_manager/security_manager/override.py b/airflow/providers/fab/auth_manager/security_manager/override.py index 73b04e0a3ab5e..3cdaf214038d2 100644 --- a/airflow/providers/fab/auth_manager/security_manager/override.py +++ b/airflow/providers/fab/auth_manager/security_manager/override.py @@ -69,7 +69,6 @@ from flask_login import LoginManager from itsdangerous import want_bytes from markupsafe import Markup -from packaging.version import Version from sqlalchemy import and_, func, inspect, literal, or_, select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload @@ -850,7 +849,8 @@ def _init_config(self): app.config.setdefault("AUTH_ROLES_SYNC_AT_LOGIN", False) app.config.setdefault("AUTH_API_LOGIN_ALLOW_MULTIPLE_PROVIDERS", False) - # Werkzeug prior to 3.0.0 does not support scrypt + from packaging.version import Version + parsed_werkzeug_version = Version(importlib.metadata.version("werkzeug")) if parsed_werkzeug_version < Version("3.0.0"): app.config.setdefault( @@ -861,9 +861,10 @@ def _init_config(self): else: app.config.setdefault( "AUTH_DB_FAKE_PASSWORD_HASH_CHECK", - "scrypt:32768:8:1$wiDa0ruWlIPhp9LM$6e409d093e62ad54df2af895d0e125b05ff6cf6414" - "8350189ffc4bcc71286edf1b8ad94a442c00f890224bf2b32153d0750c89ee9" - "401e62f9dcee5399065e4e5", + "scrypt:32768:8:1$wiDa0ruWlIPhp9LM$6e40" + "9d093e62ad54df2af895d0e125b05ff6cf6414" + "8350189ffc4bcc71286edf1b8ad94a442c00f8" + "90224bf2b32153d0750c89ee9401e62f9dcee5399065e4e5", ) # LDAP Config diff --git a/airflow/providers/fab/provider.yaml b/airflow/providers/fab/provider.yaml index f36519ff278fa..dfa4a70025c6a 100644 --- a/airflow/providers/fab/provider.yaml +++ b/airflow/providers/fab/provider.yaml @@ -62,7 +62,7 @@ dependencies: # Every time we update FAB version here, please make sure that you review the classes and models in # `airflow/providers/fab/auth_manager/security_manager/override.py` with their upstream counterparts. # In particular, make sure any breaking changes, for example any new methods, are accounted for. - - flask-appbuilder==4.5.3 + - flask-appbuilder==4.5.4 - google-re2>=1.0 - jmespath>=0.7.0 diff --git a/airflow/settings.py b/airflow/settings.py index 85d56d4be8573..259e53078d45f 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -613,6 +613,10 @@ def dispose_orm(): global engine global Session + from airflow.www.extensions import init_session + + init_session._session_interface = None + if Session is not None: # type: ignore[truthy-function] Session.remove() Session = None diff --git a/airflow/www/app.py b/airflow/www/app.py index 927d3c9b69390..79e2dcf6c4ba1 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -187,7 +187,7 @@ def create_app(config=None, testing=False): init_jinja_globals(flask_app) init_xframe_protection(flask_app) init_cache_control(flask_app) - init_airflow_session_interface(flask_app) + init_airflow_session_interface(flask_app, db) init_check_user_active(flask_app) return flask_app diff --git a/airflow/www/extensions/init_session.py b/airflow/www/extensions/init_session.py index 349990a39a2e3..c444a7c858657 100644 --- a/airflow/www/extensions/init_session.py +++ b/airflow/www/extensions/init_session.py @@ -23,7 +23,52 @@ from airflow.www.session import AirflowDatabaseSessionInterface, AirflowSecureCookieSessionInterface -def init_airflow_session_interface(app): +# Monkey patch flask-session's create_session_model to fix compatibility with Flask-SQLAlchemy 2.5.1 +# The issue is that dynamically created Session models don't inherit query_class from db.Model, +# which causes AttributeError when flask-session tries to use .query property. +# This patch ensures query_class is set on the Session model class. +def _patch_flask_session_create_session_model(): + """ + Patch flask-session's create_session_model to ensure query_class compatibility. + + This fixes the issue where flask-session's Session model doesn't have the query_class + attribute required by Flask-SQLAlchemy's _QueryProperty. + """ + try: + from flask_session.sqlalchemy import sqlalchemy as flask_session_module + + _original_create_session_model = flask_session_module.create_session_model + _session_model = None + + def patched_create_session_model(db, table_name, schema=None, bind_key=None, sequence=None): + nonlocal _session_model + if _session_model: + return _session_model + + # Create new model + Session = _original_create_session_model(db, table_name, schema, bind_key, sequence) + + # Ensure query_class is set for compatibility with Flask-SQLAlchemy + # Use db.Query which is always available on the SQLAlchemy instance + if not hasattr(Session, "query_class"): + Session.query_class = getattr(db, "Query", None) or getattr(db.Model, "query_class", None) + + _session_model = Session + return Session + + flask_session_module.create_session_model = patched_create_session_model + except ImportError: + # flask-session not installed, no need to patch + pass + + +# Apply the patch immediately when this module is imported +_patch_flask_session_create_session_model() + +_session_interface = None + + +def init_airflow_session_interface(app, sqlalchemy_client): """Set airflow session interface.""" config = app.config.copy() selected_backend = conf.get("webserver", "SESSION_BACKEND") @@ -42,9 +87,13 @@ def make_session_permanent(): app.before_request(make_session_permanent) elif selected_backend == "database": - app.session_interface = AirflowDatabaseSessionInterface( + global _session_interface + if _session_interface: + app.session_interface = _session_interface + return + _session_interface = AirflowDatabaseSessionInterface( app=app, - client=None, + client=sqlalchemy_client, permanent=permanent_cookie, # Typically these would be configurable with Flask-Session, # but we will set them explicitly instead as they don't make @@ -53,6 +102,7 @@ def make_session_permanent(): key_prefix="", use_signer=True, ) + app.session_interface = _session_interface else: raise AirflowConfigException( "Unrecognized session backend specified in " diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py index 6e18966416da8..15b54dd3ef21b 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py @@ -21,7 +21,6 @@ import os import random import re -import shutil import subprocess import sys import tempfile @@ -1192,12 +1191,6 @@ def _generate_build_files_for_provider( ) init_py_path = provider_details.base_provider_package_path / "__init__.py" init_py_path.write_text(init_py_content) - # TODO(potiuk) - remove this if when we move all providers to new structure - if provider_details.is_new_structure: - _generate_get_provider_info_py(context, provider_details) - _generate_readme_rst(context, provider_details) - _generate_pyproject(context, provider_details) - shutil.copy(AIRFLOW_SOURCES_ROOT / "LICENSE", provider_details.base_provider_package_path / "LICENSE") def _replace_min_airflow_version_in_provider_yaml( diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index ac6f7d40dee7e..9b5afdc743cab 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -55,7 +55,6 @@ SelectiveProvidersTestType, all_helm_test_packages, all_selective_core_test_types, - providers_test_type, ) from airflow_breeze.utils.console import get_console from airflow_breeze.utils.exclude_from_matrix import excluded_combos @@ -88,9 +87,7 @@ ALL_CI_SELECTIVE_TEST_TYPES = "API Always CLI Core Operators Other Serialization WWW" -ALL_PROVIDERS_SELECTIVE_TEST_TYPES = ( - "Providers[-amazon,google,standard] Providers[amazon] Providers[google] Providers[standard]" -) +ALL_PROVIDERS_SELECTIVE_TEST_TYPES = "Providers[fab]" class FileGroupForCi(Enum): @@ -653,7 +650,7 @@ def mypy_checks(self) -> list[str]: FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) or self._are_all_providers_affected() - ) and self._default_branch == "main": + ): checks_to_run.append("mypy-providers") if ( self._matching_files( @@ -767,7 +764,7 @@ def _select_test_type_if_matching( def _are_all_providers_affected(self) -> bool: # if "Providers" test is present in the list of tests, it means that we should run all providers tests # prepare all providers packages and build all providers documentation - return "Providers" in self._get_providers_test_types_to_run() + return "Providers[fab]" in self._get_providers_test_types_to_run() def _fail_if_suspended_providers_affected(self) -> bool: return "allow suspended provider changes" not in self._pr_labels @@ -841,13 +838,12 @@ def _get_core_test_types_to_run(self) -> list[str]: return sorted_candidate_test_types def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = False) -> list[str]: - if self._default_branch != "main": - return [] + # For v2-11-test branch we always run airflow + fab provider tests if self.full_tests_needed: if split_to_individual_providers: - return list(providers_test_type()) + return ["Providers[fab]"] else: - return ["Providers"] + return ["Providers[fab]"] else: all_providers_source_files = self._matching_files( FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES @@ -864,26 +860,7 @@ def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = # IF API tests are needed, that will trigger extra provider checks return [] else: - affected_providers = self._find_all_providers_affected( - include_docs=False, - ) - candidate_test_types: set[str] = set() - if isinstance(affected_providers, AllProvidersSentinel): - if split_to_individual_providers: - for provider in get_available_packages(): - candidate_test_types.add(f"Providers[{provider}]") - else: - candidate_test_types.add("Providers") - elif affected_providers: - if split_to_individual_providers: - for provider in affected_providers: - candidate_test_types.add(f"Providers[{provider}]") - else: - candidate_test_types.add(f"Providers[{','.join(sorted(affected_providers))}]") - sorted_candidate_test_types = sorted(candidate_test_types) - get_console().print("[warning]Selected providers test type candidates to run:[/]") - get_console().print(sorted_candidate_test_types) - return sorted_candidate_test_types + return ["fab"] @staticmethod def _extract_long_provider_tests(current_test_types: set[str]): @@ -930,7 +907,8 @@ def providers_test_types_list_as_string(self) -> str | None: if self._default_branch != "main": test_types_to_remove: set[str] = set() for test_type in current_test_types: - if test_type.startswith("Providers"): + # For v2-11-test branch we always run airflow + fab provider tests + if test_type.startswith("Providers") and not test_type.startswith("Providers[fab]"): get_console().print( f"[warning]Removing {test_type} because the target branch " f"is {self._default_branch} and not main[/]" @@ -1072,7 +1050,8 @@ def docs_list_as_string(self) -> str | None: if not self.docs_build: return None if self._default_branch != "main": - return "apache-airflow docker-stack" + # For v2-11-test branch we always run airflow + fab provider tests + return "apache-airflow docker-stack fab" if self.full_tests_needed: return _ALL_DOCS_LIST providers_affected = self._find_all_providers_affected( @@ -1164,8 +1143,7 @@ def skip_pre_commits(self) -> str: @cached_property def skip_providers_tests(self) -> bool: - if self._default_branch != "main": - return True + # For v2-11-test branch we always run airflow + fab provider tests if self.full_tests_needed: return False if self._get_providers_test_types_to_run(): @@ -1200,18 +1178,21 @@ def helm_test_packages(self) -> str: @cached_property def selected_providers_list_as_string(self) -> str | None: - if self._default_branch != "main": - return None - if self.full_tests_needed: - return "" - if self._are_all_providers_affected(): - return "" - affected_providers = self._find_all_providers_affected(include_docs=True) - if not affected_providers: - return None - if isinstance(affected_providers, AllProvidersSentinel): - return "" - return " ".join(sorted(affected_providers)) + # For v2-11-test branch we always test airflow + fab provider + return "fab" + + # if self._default_branch != "main": + # return None + # if self.full_tests_needed: + # return "" + # if self._are_all_providers_affected(): + # return "" + # affected_providers = self._find_all_providers_affected(include_docs=True) + # if not affected_providers: + # return None + # if isinstance(affected_providers, AllProvidersSentinel): + # return "" + # return " ".join(sorted(affected_providers)) @cached_property def runs_on_as_json_default(self) -> str: diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index fd0cb32391cc1..81df4011ba1e6 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -157,7 +157,7 @@ def test_find_matching_long_package_name_bad_filter(): """ "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow>=2.9.0", - "flask-appbuilder==4.5.3", + "flask-appbuilder==4.5.4", "flask-login>=0.6.3", "flask-session>=0.8.0", "flask>=2.2,<3", @@ -172,7 +172,7 @@ def test_find_matching_long_package_name_bad_filter(): """ "apache-airflow-providers-common-compat>=1.2.1.dev0", "apache-airflow>=2.9.0.dev0", - "flask-appbuilder==4.5.3", + "flask-appbuilder==4.5.4", "flask-login>=0.6.3", "flask-session>=0.8.0", "flask>=2.2,<3", @@ -187,7 +187,7 @@ def test_find_matching_long_package_name_bad_filter(): """ "apache-airflow-providers-common-compat>=1.2.1b0", "apache-airflow>=2.9.0b0", - "flask-appbuilder==4.5.3", + "flask-appbuilder==4.5.4", "flask-login>=0.6.3", "flask-session>=0.8.0", "flask>=2.2,<3", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 604aec1dcc4f8..be8febcc70f8e 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -30,7 +30,6 @@ GithubEvents, ) from airflow_breeze.utils.functools_cache import clearable_cache -from airflow_breeze.utils.packages import get_available_packages from airflow_breeze.utils.selective_checks import ( ALL_CI_SELECTIVE_TEST_TYPES, ALL_PROVIDERS_SELECTIVE_TEST_TYPES, @@ -41,10 +40,8 @@ ALL_DOCS_SELECTED_FOR_BUILD = "" -ALL_PROVIDERS_AFFECTED = "" -LIST_OF_ALL_PROVIDER_TESTS = " ".join( - f"Providers[{provider}]" for provider in get_available_packages(include_not_ready=True) -) +ALL_PROVIDERS_AFFECTED = "fab" +LIST_OF_ALL_PROVIDER_TESTS = "Providers[fab]" # commit that is neutral - allows to keep pyproject.toml-changing PRS neutral for unit tests @@ -295,7 +292,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "prod-image-build": "true", "needs-helm-tests": "true", "run-tests": "true", - "run-amazon-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "full-tests-needed": "true", "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", @@ -322,7 +319,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "prod-image-build": "true", "needs-helm-tests": "true", "run-tests": "true", - "run-amazon-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "full-tests-needed": "true", "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", @@ -348,7 +345,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "prod-image-build": "true", "needs-helm-tests": "true", "run-tests": "true", - "run-amazon-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "full-tests-needed": "true", "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", @@ -374,7 +371,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "prod-image-build": "true", "needs-helm-tests": "true", "run-tests": "true", - "run-amazon-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "full-tests-needed": "true", "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", @@ -783,19 +780,20 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "ci-image-build": "true", "prod-image-build": "true", "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", + "skip-providers-tests": "false", + "providers-test-types-list-as-string": "Providers[fab]", + "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow docker-stack", + "docs-list-as-string": "apache-airflow docker-stack fab", "full-tests-needed": "true", "skip-pre-commits": "check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,kubeconform,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,validate-operators-init", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": "API Always CLI Core Operators Other " "Serialization WWW", "needs-mypy": "true", - "mypy-checks": "['mypy-airflow', 'mypy-docs', 'mypy-dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev']", }, - id="Everything should run except Providers and lint pre-commit " + id="Everything should run including Providers[fab] except lint pre-commit " "when full tests are needed for non-main branch", ) ), @@ -853,16 +851,16 @@ def test_expected_output_full_tests_needed( "ci-image-build": "true", "prod-image-build": "true", "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow docker-stack", + "docs-list-as-string": "apache-airflow docker-stack fab", "full-tests-needed": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": "Always", - "needs-mypy": "false", - "mypy-checks": "[]", + "needs-mypy": "true", + "mypy-checks": "['mypy-providers']", }, id="No Helm tests, No providers no lint charts, should run if " "only chart/providers changed in non-main but PROD image should be built", @@ -880,16 +878,16 @@ def test_expected_output_full_tests_needed( "prod-image-build": "true", "needs-helm-tests": "false", "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow docker-stack", + "docs-list-as-string": "apache-airflow docker-stack fab", "full-tests-needed": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": "Always CLI", "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", }, id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch", ), @@ -905,16 +903,16 @@ def test_expected_output_full_tests_needed( "prod-image-build": "false", "needs-helm-tests": "false", "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow docker-stack", + "docs-list-as-string": "apache-airflow docker-stack fab", "full-tests-needed": "false", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW", "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", }, id="All tests except Providers and helm lint pre-commit " "should run if core file changed in non-main branch", @@ -973,11 +971,11 @@ def test_expected_output_pull_request_v2_7( "run-tests": "true", "docs-build": "true", "skip-pre-commits": "check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,kubeconform,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,validate-operators-init", - "docs-list-as-string": "apache-airflow docker-stack", + "docs-list-as-string": "apache-airflow docker-stack fab", "upgrade-to-newer-dependencies": "true", "core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW", "needs-mypy": "true", - "mypy-checks": "['mypy-airflow', 'mypy-docs', 'mypy-dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev']", }, id="All tests except Providers and Helm run on push" " even if unimportant file changed in non-main branch", diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index fa0ee5b6e85d5..80b33d188b501 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -40,8 +40,7 @@ rm -fv ./dist/* ./docker-context-files/* breeze release-management prepare-provider-packages \ --package-list-file ./prod_image_installed_providers.txt \ - --package-format wheel \ - --version-suffix-for-pypi dev0 + --package-format wheel breeze release-management prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 diff --git a/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst b/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst index 48c8c8f1b1f25..d702d76e191fb 100644 --- a/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst +++ b/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst @@ -64,7 +64,7 @@ methods like OAuth, OpenID, LDAP, REMOTE_USER. It should be noted that due to th and Authlib, only a selection of OAuth2 providers is supported. This list includes ``github``, ``githublocal``, ``twitter``, ``linkedin``, ``google``, ``azure``, ``openshift``, ``okta``, ``keycloak`` and ``keycloak_before_17``. -The default authentication option described in the :ref:`Web Authentication ` section is related +The default authentication option described in the Web Authentication section of Airflow 2 docs is related with the following entry in the ``$AIRFLOW_HOME/webserver_config.py``. .. code-block:: ini diff --git a/docs/apache-airflow-providers-fab/changelog.rst b/docs/apache-airflow-providers-fab/changelog.rst index c6bdcaa11e75a..390f94b034dae 100644 --- a/docs/apache-airflow-providers-fab/changelog.rst +++ b/docs/apache-airflow-providers-fab/changelog.rst @@ -16,8 +16,7 @@ specific language governing permissions and limitations under the License. - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY diff --git a/docs/apache-airflow-providers-fab/commits.rst b/docs/apache-airflow-providers-fab/commits.rst index 032c64b24da7b..87fe5c4f91759 100644 --- a/docs/apache-airflow-providers-fab/commits.rst +++ b/docs/apache-airflow-providers-fab/commits.rst @@ -16,13 +16,12 @@ specific language governing permissions and limitations under the License. - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! Package apache-airflow-providers-fab ------------------------------------------------------ @@ -35,14 +34,77 @@ For high-level changelog, see :doc:`package information including changelog `_ 2024-10-09 ``Split providers out of the main "airflow/" tree into a UV workspace project (#42505)`` +================================================================================================= =========== ======================================================================================== + +1.4.1 +..... + +Latest change: 2024-10-09 + +================================================================================================= =========== ================================================================================================================================ +Commit Committed Subject +================================================================================================= =========== ================================================================================================================================ +`2bb8628463 `_ 2024-10-09 ``Prepare docs for Oct 1st adhoc wave of providers (#42862)`` +`9536c98a43 `_ 2024-10-01 ``Update Rest API tests to no longer rely on FAB auth manager. Move tests specific to FAB permissions to FAB provider (#42523)`` +`ede7cb27fd `_ 2024-09-30 ``Rename dataset related python variable names to asset (#41348)`` +`2beb6a765d `_ 2024-09-25 ``Simplify expression for get_permitted_dag_ids query (#42484)`` +================================================================================================= =========== ================================================================================================================================ + +1.4.0 +..... + +Latest change: 2024-09-21 + +================================================================================================= =========== =================================================================================== +Commit Committed Subject +================================================================================================= =========== =================================================================================== +`7628d47d04 `_ 2024-09-21 ``Prepare docs for Sep 1st wave of providers (#42387)`` +`6a527c9fac `_ 2024-09-21 ``Fix pre-commit for auto update of fab migration versions (#42382)`` +`8741e9c176 `_ 2024-09-20 ``Handle 'AUTH_ROLE_PUBLIC' in FAB auth manager (#42280)`` +`9f167bbc34 `_ 2024-09-19 ``Add FAB migration commands (#41804)`` +`db7f92787a `_ 2024-09-17 ``Deprecated kerberos auth removed (#41693)`` +`d1e500c450 `_ 2024-09-16 ``Deprecated configuration removed (#42129)`` +`a094f9105c `_ 2024-09-12 ``Move 'is_active' user property to FAB auth manager (#42042)`` +`7b6eb92537 `_ 2024-09-04 ``Move 'register_views' to auth manager interface (#41777)`` +`1379376b66 `_ 2024-09-02 ``Add TODOs in providers code for Subdag code removal (#41963)`` +`f16107017c `_ 2024-09-02 ``Revert "Provider fab auth manager deprecated methods removed (#41720)" (#41960)`` +`b0391838c1 `_ 2024-08-26 ``Provider fab auth manager deprecated methods removed (#41720)`` +`59dc98178b `_ 2024-08-25 ``Separate FAB migration from Core Airflow migration (#41437)`` +`c78a004210 `_ 2024-08-20 ``Add fixes by breeze/precommit-lint static checks (#41604) (#41618)`` +`d6df0786cf `_ 2024-08-20 ``Make kerberos an optional and devel dependency for impala and fab (#41616)`` +================================================================================================= =========== =================================================================================== + +1.3.0 +..... + +Latest change: 2024-08-19 + +================================================================================================= =========== ========================================================================== +Commit Committed Subject +================================================================================================= =========== ========================================================================== +`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` +`6570c6d1bb `_ 2024-08-13 ``Remove deprecated SubDags (#41390)`` +`090607d92a `_ 2024-08-08 ``Feature: Allow set Dag Run resource into Dag Level permission (#40703)`` +================================================================================================= =========== ========================================================================== + 1.2.2 ..... -Latest change: 2024-07-25 +Latest change: 2024-07-28 ================================================================================================= =========== ===================================================================================== Commit Committed Subject ================================================================================================= =========== ===================================================================================== +`7126678e87 `_ 2024-07-28 ``Prepare Providers docs ad hoc release (#41074)`` `95cab23792 `_ 2024-07-25 ``Bug fix: sync perm command not able to use custom security manager (#41020)`` `6684481c67 `_ 2024-07-20 ``AIP-44 make database isolation mode work in Breeze (#40894)`` `d029e77f2f `_ 2024-07-15 ``Bump version checked by FAB provider on logout CSRF protection to 2.10.0 (#40784)`` diff --git a/docs/apache-airflow-providers-fab/index.rst b/docs/apache-airflow-providers-fab/index.rst index 93da1fa933366..a0928e5ae383c 100644 --- a/docs/apache-airflow-providers-fab/index.rst +++ b/docs/apache-airflow-providers-fab/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-fab package `Flask App Builder `__ -Release: 1.2.2 +Release: 1.5.4 Provider package ---------------- @@ -96,13 +96,43 @@ Requirements The minimum Apache Airflow version supported by this provider package is ``2.9.0``. -==================== ================== -PIP package Version required -==================== ================== -``apache-airflow`` ``>=2.9.0`` -``flask`` ``>=2.2,<2.3`` -``flask-appbuilder`` ``==4.5.0`` -``flask-login`` ``>=0.6.2`` -``google-re2`` ``>=1.0`` -``jmespath`` ``>=0.7.0`` -==================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=2.9.0`` +``apache-airflow-providers-common-compat`` ``>=1.2.1`` +``flask-login`` ``>=0.6.3`` +``flask-session`` ``>=0.8.0`` +``flask`` ``>=2.2,<3`` +``flask-appbuilder`` ``==4.5.4`` +``google-re2`` ``>=1.0`` +``jmespath`` ``>=0.7.0`` +========================================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-fab[common.compat] + + +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +================================================================================================================== ================= + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-fab 1.5.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-fab 1.5.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index b02d1d211b515..8a2664d203b81 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -550,7 +550,7 @@ "deps": [ "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow>=2.9.0", - "flask-appbuilder==4.5.3", + "flask-appbuilder==4.5.4", "flask-login>=0.6.3", "flask-session>=0.8.0", "flask>=2.2,<3", diff --git a/hatch_build.py b/hatch_build.py index 494ada007716f..44238045fb3d2 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -161,6 +161,7 @@ "checksumdir>=1.2.0; python_version >= '3.9'", "click>=8.1.8; python_version >= '3.9'", "docutils>=0.21; python_version >= '3.9'", + "setuptools!=82.0.0", # until https://github.com/sphinx-contrib/redoc/issues/53 is resolved "sphinx-airflow-theme>=0.1.0; python_version >= '3.9'", "sphinx-argparse>=0.4.0; python_version >= '3.9'", "sphinx-autoapi>=3; python_version >= '3.9'", @@ -436,7 +437,7 @@ "fsspec>=2023.10.0", 'google-re2>=1.0;python_version<"3.12"', 'google-re2>=1.1;python_version>="3.12"', - "gunicorn>=20.1.0", + "gunicorn>=21.2.0", "httpx>=0.25.0", 'importlib_metadata>=6.5;python_version<"3.12"', # Importib_resources 6.2.0-6.3.1 break pytest_rewrite diff --git a/scripts/ci/constraints/ci_branch_constraints.sh b/scripts/ci/constraints/ci_branch_constraints.sh index d4f73e62e591a..a99759901b8e2 100755 --- a/scripts/ci/constraints/ci_branch_constraints.sh +++ b/scripts/ci/constraints/ci_branch_constraints.sh @@ -16,14 +16,7 @@ # specific language governing permissions and limitations # under the License. # shellcheck disable=SC2086 -if [[ ${GITHUB_REF} == 'refs/heads/main' ]]; then - echo "branch=constraints-main" -elif [[ ${GITHUB_REF} =~ refs/heads/v([0-9\-]*)\-(test|stable) ]]; then - echo "branch=constraints-${BASH_REMATCH[1]}" -else - # Assume PR to constraints-main here - echo >&2 - echo "[${COLOR_YELLOW}Assuming that the PR is to 'main' branch!${COLOR_RESET}" >&2 - echo >&2 - echo "branch=constraints-main" -fi +echo >&2 +echo "[${COLOR_YELLOW}Hard-code constraints-2-11!${COLOR_RESET}" >&2 +echo >&2 +echo "branch=constraints-2-11" diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py index 3d5c743b54f78..fdc9162143bc9 100755 --- a/scripts/ci/pre_commit/check_system_tests.py +++ b/scripts/ci/pre_commit/check_system_tests.py @@ -38,13 +38,13 @@ WATCHER_APPEND_INSTRUCTION_SHORT = " >> watcher()" PYTEST_FUNCTION = """ -from tests_common.test_utils.system_tests import get_test_run # noqa: E402 +from tests.test_utils.system_tests import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) """ PYTEST_FUNCTION_PATTERN = re.compile( - r"from tests_common\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" + r"from tests\.test_utils\.system_tests import get_test_run(?: # noqa: E402)?\s+" r"(?:# .+\))?\s+" r"test_run = get_test_run\(dag\)" ) @@ -52,7 +52,7 @@ def _check_file(file: Path): content = file.read_text() - if "from tests_common.test_utils.watcher import watcher" in content: + if "from tests.test_utils.watcher import watcher" in content: index = content.find(WATCHER_APPEND_INSTRUCTION_SHORT) if index == -1: errors.append( diff --git a/scripts/ci/pre_commit/mypy_folder.py b/scripts/ci/pre_commit/mypy_folder.py index a1f4aa337babc..db48d6f010aff 100755 --- a/scripts/ci/pre_commit/mypy_folder.py +++ b/scripts/ci/pre_commit/mypy_folder.py @@ -33,7 +33,7 @@ ALLOWED_FOLDERS = [ "airflow", - "providers/src/airflow/providers", + "airflow/providers/fab", "dev", "docs", ] @@ -48,13 +48,9 @@ sys.exit(1) arguments = [mypy_folder] -if mypy_folder == "providers/src/airflow/providers": - arguments.extend( - [ - "providers/tests", - "--namespace-packages", - ] - ) +script = "/opt/airflow/scripts/in_container/run_mypy.sh" +if mypy_folder == "airflow/providers/fab": + script = "/opt/airflow/scripts/in_container/run_mypy_providers.sh" if mypy_folder == "airflow": arguments.extend( @@ -63,11 +59,11 @@ ] ) -print("Running /opt/airflow/scripts/in_container/run_mypy.sh with arguments: ", arguments) +print(f"Running {script} with arguments: {arguments}") res = run_command_via_breeze_shell( [ - "/opt/airflow/scripts/in_container/run_mypy.sh", + script, *arguments, ], warn_image_upgrade_needed=True, diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index c1123303d8eb7..312cca60d8466 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -390,7 +390,7 @@ function check_force_lowest_dependencies() { EXTRA="[devel]" if [[ ${TEST_TYPE=} =~ Providers\[.*\] ]]; then # shellcheck disable=SC2001 - EXTRA=$(echo "[${TEST_TYPE},devel]" | sed 's/Providers\[\(.*\)\]/\1/') + EXTRA=$(echo "[${TEST_TYPE}]" | sed 's/Providers\[\([^]]*\)\]/\1,devel/') echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${EXTRA}${COLOR_RESET}" echo diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index 16dfb9279d0f9..9afaeea87f98e 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -59,8 +59,9 @@ def find_airflow_package(extension: str) -> str | None: def find_provider_packages(extension: str, selected_providers: list[str]) -> list[str]: candidates = list(DIST_FOLDER.glob(f"apache_airflow_providers_*.{extension}")) console.print("\n[bright_blue]Found the following provider packages: ") + console.print(f"Filtering by {selected_providers}") for candidate in sorted(candidates): - console.print(f" {candidate.as_posix()}") + console.print(f" {candidate.as_posix()} -> {get_provider_name(candidate.name)}") console.print() if selected_providers: candidates = [ @@ -272,7 +273,7 @@ def find_installation_spec( ) provider_package_list = [] if use_packages_from_dist: - selected_providers_list = install_selected_providers.split(",") if install_selected_providers else [] + selected_providers_list = install_selected_providers.split(" ") if install_selected_providers else [] if selected_providers_list: console.print(f"\n[bright_blue]Selected providers: {selected_providers_list}\n") else: diff --git a/scripts/in_container/run_mypy_providers.sh b/scripts/in_container/run_mypy_providers.sh new file mode 100755 index 0000000000000..462d9a4e96c2b --- /dev/null +++ b/scripts/in_container/run_mypy_providers.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Script to run mypy on all code. Can be started from any working directory +# shellcheck source=scripts/in_container/_in_container_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" +export PYTHONPATH=${AIRFLOW_SOURCES} + +export MYPY_FORCE_COLOR=true +export TERM=ansi + +mypy --namespace-packages "${@}" diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py index a7a97d78ca4a2..3b94512b0c7ca 100755 --- a/scripts/in_container/verify_providers.py +++ b/scripts/in_container/verify_providers.py @@ -132,7 +132,7 @@ class ProviderPackageDetails(NamedTuple): def get_all_providers() -> list[str]: - return list(ALL_DEPENDENCIES.keys()) + return ["fab"] def import_all_classes( diff --git a/tests/conftest.py b/tests/conftest.py index 6c17c6f4036d4..09a055bdcc039 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -371,10 +371,17 @@ def initial_db_init(): from airflow.www.extensions.init_auth_manager import get_auth_manager from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_10_PLUS - if AIRFLOW_V_2_10_PLUS: - db.resetdb(use_migration_files=True) + sql_alchemy_conn = conf.get("database", "sql_alchemy_conn") + if sql_alchemy_conn.startswith("sqlite"): + reset_cmd = [sys.executable, "-m", "airflow", "db", "reset", "--yes"] + if AIRFLOW_V_2_10_PLUS: + reset_cmd.append("--use-migration-files") + subprocess.check_call(reset_cmd) else: - db.resetdb() + if AIRFLOW_V_2_10_PLUS: + db.resetdb(use_migration_files=True) + else: + db.resetdb() db.bootstrap_dagbag() # minimal app to add roles flask_app = Flask(__name__) diff --git a/tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py b/tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py index e57f34ce4b033..c763042e1c955 100644 --- a/tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py +++ b/tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth import init_app diff --git a/tests/providers/fab/auth_manager/api/auth/backend/test_session.py b/tests/providers/fab/auth_manager/api/auth/backend/test_session.py index 405eafe11dfc4..4e7eb3a3ae389 100644 --- a/tests/providers/fab/auth_manager/api/auth/backend/test_session.py +++ b/tests/providers/fab/auth_manager/api/auth/backend/test_session.py @@ -20,10 +20,10 @@ import pytest from flask import Response -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from airflow.providers.fab.auth_manager.api.auth.backend.session import requires_authentication from airflow.www import app as application +from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py b/tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py index b208b845096b9..61d923d5ff125 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py +++ b/tests/providers/fab/auth_manager/api_endpoints/api_connexion_utils.py @@ -18,7 +18,7 @@ from contextlib import contextmanager -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_asset_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_asset_endpoint.py deleted file mode 100644 index 4cd76aa2b4a54..0000000000000 --- a/tests/providers/fab/auth_manager/api_endpoints/test_asset_endpoint.py +++ /dev/null @@ -1,327 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -from typing import Generator - -import pytest -import time_machine -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_assets, clear_db_runs -from tests_common.test_utils.www import _check_last_log - -from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP -from airflow.security import permissions -from airflow.utils import timezone - -try: - from airflow.models.asset import AssetDagRunQueue, AssetModel -except ImportError: - if AIRFLOW_V_3_0_PLUS: - raise - else: - pass - -pytestmark = [ - pytest.mark.db_test, - pytest.mark.skip_if_database_isolation_mode, - pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="Test requires Airflow 3.0+"), -] - - -@pytest.fixture(scope="module") -def configured_app(minimal_app_for_auth_api): - app = minimal_app_for_auth_api - create_user( - app, - username="test_queued_event", - role_name="TestQueuedEvent", - permissions=[ - (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), - (permissions.ACTION_CAN_READ, permissions.RESOURCE_ASSET), - (permissions.ACTION_CAN_DELETE, permissions.RESOURCE_ASSET), - ], - ) - - yield app - - delete_user(app, username="test_queued_event") - - -class TestAssetEndpoint: - default_time = "2020-06-11T18:00:00+00:00" - - @pytest.fixture(autouse=True) - def setup_attrs(self, configured_app) -> None: - self.app = configured_app - self.client = self.app.test_client() - clear_db_assets() - clear_db_runs() - - def teardown_method(self) -> None: - clear_db_assets() - clear_db_runs() - - def _create_asset(self, session): - asset_model = AssetModel( - id=1, - uri="s3://bucket/key", - extra={"foo": "bar"}, - created_at=timezone.parse(self.default_time), - updated_at=timezone.parse(self.default_time), - ) - session.add(asset_model) - session.commit() - return asset_model - - -class TestQueuedEventEndpoint(TestAssetEndpoint): - @pytest.fixture - def time_freezer(self) -> Generator: - freezer = time_machine.travel(self.default_time, tick=False) - freezer.start() - - yield - - freezer.stop() - - def _create_asset_dag_run_queues(self, dag_id, asset_id, session): - ddrq = AssetDagRunQueue(target_dag_id=dag_id, asset_id=asset_id) - session.add(ddrq) - session.commit() - return ddrq - - -class TestGetDagAssetQueuedEvent(TestQueuedEventEndpoint): - @pytest.mark.usefixtures("time_freezer") - def test_should_respond_200(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - asset_uri = "s3://bucket/key" - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 200 - assert response.json == { - "created_at": self.default_time, - "uri": "s3://bucket/key", - "dag_id": "dag", - } - - def test_should_respond_404(self): - dag_id = "not_exists" - asset_uri = "not_exists" - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` and asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - - -class TestDeleteDagAssetQueuedEvent(TestAssetEndpoint): - def test_delete_should_respond_204(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_uri = "s3://bucket/key" - asset_id = self._create_asset(session).id - - ddrq = AssetDagRunQueue(target_dag_id=dag_id, asset_id=asset_id) - session.add(ddrq) - session.commit() - conn = session.query(AssetDagRunQueue).all() - assert len(conn) == 1 - - response = self.client.delete( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 204 - conn = session.query(AssetDagRunQueue).all() - assert len(conn) == 0 - _check_last_log( - session, dag_id=dag_id, event="api.delete_dag_asset_queued_event", execution_date=None - ) - - def test_should_respond_404(self): - dag_id = "not_exists" - asset_uri = "not_exists" - - response = self.client.delete( - f"/api/v1/dags/{dag_id}/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` and asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - - -class TestGetDagAssetQueuedEvents(TestQueuedEventEndpoint): - @pytest.mark.usefixtures("time_freezer") - def test_should_respond_200(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 200 - assert response.json == { - "queued_events": [ - { - "created_at": self.default_time, - "uri": "s3://bucket/key", - "dag_id": "dag", - } - ], - "total_entries": 1, - } - - def test_should_respond_404(self): - dag_id = "not_exists" - - response = self.client.get( - f"/api/v1/dags/{dag_id}/assets/queuedEvent", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - - -class TestDeleteDagDatasetQueuedEvents(TestAssetEndpoint): - def test_should_respond_404(self): - dag_id = "not_exists" - - response = self.client.delete( - f"/api/v1/dags/{dag_id}/assets/queuedEvent", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with dag_id: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - - -class TestGetDatasetQueuedEvents(TestQueuedEventEndpoint): - @pytest.mark.usefixtures("time_freezer") - def test_should_respond_200(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - asset_uri = "s3://bucket/key" - - response = self.client.get( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 200 - assert response.json == { - "queued_events": [ - { - "created_at": self.default_time, - "uri": "s3://bucket/key", - "dag_id": "dag", - } - ], - "total_entries": 1, - } - - def test_should_respond_404(self): - asset_uri = "not_exists" - - response = self.client.get( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json - - -class TestDeleteDatasetQueuedEvents(TestQueuedEventEndpoint): - def test_delete_should_respond_204(self, session, create_dummy_dag): - dag, _ = create_dummy_dag() - dag_id = dag.dag_id - asset_id = self._create_asset(session).id - self._create_asset_dag_run_queues(dag_id, asset_id, session) - asset_uri = "s3://bucket/key" - - response = self.client.delete( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 204 - conn = session.query(AssetDagRunQueue).all() - assert len(conn) == 0 - _check_last_log(session, dag_id=None, event="api.delete_asset_queued_events", execution_date=None) - - def test_should_respond_404(self): - asset_uri = "not_exists" - - response = self.client.delete( - f"/api/v1/assets/queuedEvent/{asset_uri}", - environ_overrides={"REMOTE_USER": "test_queued_event"}, - ) - - assert response.status_code == 404 - assert { - "detail": "Queue event with asset uri: `not_exists` was not found", - "status": 404, - "title": "Queue event not found", - "type": EXCEPTIONS_LINK_MAP[404], - } == response.json diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_auth.py b/tests/providers/fab/auth_manager/api_endpoints/test_auth.py index 6a90b7ec4b30e..fcf3f3cb6ccf9 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_auth.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_auth.py @@ -20,11 +20,12 @@ import pytest from flask_login import current_user -from tests_common.test_utils.api_connexion_utils import assert_401 -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_pools -from tests_common.test_utils.www import client_with_login + +from tests.test_utils.api_connexion_utils import assert_401 +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.config import conf_vars +from tests.test_utils.db import clear_db_pools +from tests.test_utils.www import client_with_login pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_cors.py b/tests/providers/fab/auth_manager/api_endpoints/test_cors.py index 3741d71fb8b96..b44eab8820ec6 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_cors.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_cors.py @@ -19,9 +19,10 @@ from base64 import b64encode import pytest -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.config import conf_vars -from tests_common.test_utils.db import clear_db_pools + +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.config import conf_vars +from tests.test_utils.db import clear_db_pools pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py index 198f213aa25ff..5b1049f54aa54 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_dag_endpoint.py @@ -21,10 +21,6 @@ import pendulum import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags -from tests_common.test_utils.www import _check_last_log from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagBag, DagModel @@ -32,6 +28,10 @@ from airflow.operators.empty import EmptyOperator from airflow.security import permissions from airflow.utils.session import provide_session +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_serialized_dags +from tests.test_utils.www import _check_last_log pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py index 9eab53aaa0cd4..f0d9b0da298c6 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_dag_source_endpoint.py @@ -21,12 +21,12 @@ from typing import TYPE_CHECKING import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags from airflow.models import DagBag from airflow.security import permissions +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_dag_code, clear_db_dags, clear_db_serialized_dags pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py index b42d92d9cacbd..adfde1cc5b3eb 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_dag_warning_endpoint.py @@ -17,14 +17,14 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dag_warnings, clear_db_dags from airflow.models.dag import DagModel from airflow.models.dagwarning import DagWarning from airflow.security import permissions from airflow.utils.session import create_session +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_dag_warnings, clear_db_dags pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py index 225a79bd9ac67..acf3ca62684a1 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_event_log_endpoint.py @@ -17,13 +17,13 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_logs from airflow.models import Log from airflow.security import permissions from airflow.utils import timezone +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_logs pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py index 5bac8356af59a..a2fa1d028a3f2 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_import_error_endpoint.py @@ -17,14 +17,14 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError -from tests_common.test_utils.db import clear_db_dags, clear_db_import_errors -from tests_common.test_utils.permissions import _resource_name from airflow.models.dag import DagModel from airflow.security import permissions from airflow.utils import timezone +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, ParseImportError +from tests.test_utils.db import clear_db_dags, clear_db_import_errors +from tests.test_utils.permissions import _resource_name pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py index da3e9a06565d7..be165e2498ff1 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py @@ -17,17 +17,17 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( + +from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP +from airflow.security import permissions +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_role, create_user, delete_role, delete_user, ) -from tests_common.test_utils.api_connexion_utils import assert_401 -from tests_common.test_utils.compat import ignore_provider_compatibility_error - -from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP -from airflow.security import permissions +from tests.test_utils.api_connexion_utils import assert_401 +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import Role diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py index aaae228998e6e..72667bd343c26 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py @@ -20,13 +20,6 @@ import urllib import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user, - delete_roles, - delete_user, -) -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_runs, clear_rendered_ti_fields from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.models import DagRun, TaskInstance @@ -35,6 +28,13 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( + create_user, + delete_roles, + delete_user, +) +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_runs, clear_rendered_ti_fields pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py index eb1226d714e61..df32c7d814f58 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py @@ -19,20 +19,20 @@ import unittest.mock import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user, - delete_role, - delete_user, -) from sqlalchemy.sql.functions import count -from tests_common.test_utils.api_connexion_utils import assert_401 -from tests_common.test_utils.compat import ignore_provider_compatibility_error -from tests_common.test_utils.config import conf_vars from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP from airflow.security import permissions from airflow.utils import timezone from airflow.utils.session import create_session +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( + create_user, + delete_role, + delete_user, +) +from tests.test_utils.api_connexion_utils import assert_401 +from tests.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.config import conf_vars with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import User diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py index 920869d39e346..a8e71e1a82466 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_variable_endpoint.py @@ -17,12 +17,12 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_variables from airflow.models import Variable from airflow.security import permissions +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_variables pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py b/tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py index f0ec606038e1d..2049463cd642a 100644 --- a/tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py +++ b/tests/providers/fab/auth_manager/api_endpoints/test_xcom_endpoint.py @@ -19,9 +19,6 @@ from datetime import timedelta import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS -from tests_common.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom from airflow.models.dag import DagModel from airflow.models.dagrun import DagRun @@ -32,6 +29,9 @@ from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.types import DagRunType +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom pytestmark = [ pytest.mark.db_test, diff --git a/tests/providers/fab/auth_manager/cli_commands/test_db_command.py b/tests/providers/fab/auth_manager/cli_commands/test_db_command.py index 030b251a55ad6..6f0453c0b6b94 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_db_command.py +++ b/tests/providers/fab/auth_manager/cli_commands/test_db_command.py @@ -21,6 +21,7 @@ import pytest from airflow.cli import cli_parser +from airflow.exceptions import AirflowOptionalProviderFeatureException pytestmark = [pytest.mark.db_test] try: @@ -130,5 +131,5 @@ def test_cli_upgrade_success(self, mock_upgradedb, args, called_with): def test_cli_migratedb_failure(self, mock_upgradedb, args, pattern): with pytest.raises(SystemExit, match=pattern): db_command.migratedb(self.parser.parse_args(["fab-db", "migrate", *args])) -except (ModuleNotFoundError, ImportError): +except (ModuleNotFoundError, ImportError, AirflowOptionalProviderFeatureException): pass diff --git a/tests/providers/fab/auth_manager/cli_commands/test_definition.py b/tests/providers/fab/auth_manager/cli_commands/test_definition.py index 572cbee05e3db..2db5d352ecc19 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_definition.py +++ b/tests/providers/fab/auth_manager/cli_commands/test_definition.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.definition import ( diff --git a/tests/providers/fab/auth_manager/cli_commands/test_role_command.py b/tests/providers/fab/auth_manager/cli_commands/test_role_command.py index d07cfc61242fe..5f12c01860d1f 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_role_command.py +++ b/tests/providers/fab/auth_manager/cli_commands/test_role_command.py @@ -23,10 +23,10 @@ from typing import TYPE_CHECKING import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error -from tests_common.test_utils.config import conf_vars from airflow.cli import cli_parser +from tests.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.config import conf_vars with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import role_command diff --git a/tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py b/tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py index a0345909dc2df..9e1817bd5617c 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py +++ b/tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py @@ -20,9 +20,9 @@ from unittest import mock import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.cli import cli_parser +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import sync_perm_command diff --git a/tests/providers/fab/auth_manager/cli_commands/test_user_command.py b/tests/providers/fab/auth_manager/cli_commands/test_user_command.py index 6ccd4c99716ab..b8ce2f48d6c03 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_user_command.py +++ b/tests/providers/fab/auth_manager/cli_commands/test_user_command.py @@ -24,9 +24,9 @@ from io import StringIO import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.cli import cli_parser +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands import user_command diff --git a/tests/providers/fab/auth_manager/cli_commands/test_utils.py b/tests/providers/fab/auth_manager/cli_commands/test_utils.py index 394beb88290ed..e9abdd470b057 100644 --- a/tests/providers/fab/auth_manager/cli_commands/test_utils.py +++ b/tests/providers/fab/auth_manager/cli_commands/test_utils.py @@ -19,14 +19,14 @@ import os import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error -from tests_common.test_utils.config import conf_vars import airflow from airflow.configuration import conf from airflow.exceptions import AirflowConfigException from airflow.www.extensions.init_appbuilder import AirflowAppBuilder from airflow.www.session import AirflowDatabaseSessionInterface +from tests.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.config import conf_vars with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder diff --git a/tests/providers/fab/auth_manager/conftest.py b/tests/providers/fab/auth_manager/conftest.py index d3e14c6a520cd..671a1c16b9b40 100644 --- a/tests/providers/fab/auth_manager/conftest.py +++ b/tests/providers/fab/auth_manager/conftest.py @@ -29,6 +29,7 @@ def minimal_app_for_auth_api(): skip_all_except=[ "init_appbuilder", "init_api_auth", + "init_api_experimental_auth", "init_api_auth_provider", "init_api_connexion", "init_api_error_handlers", @@ -42,7 +43,7 @@ def factory(): ( "api", "auth_backends", - ): "providers.tests.fab.auth_manager.api_endpoints.remote_user_api_auth_backend,airflow.providers.fab.auth_manager.api.auth.backend.session", + ): "tests.test_utils.remote_user_api_auth_backend,airflow.providers.fab.auth_manager.api.auth.backend.session", ( "core", "auth_manager", diff --git a/tests/providers/fab/auth_manager/decorators/test_auth.py b/tests/providers/fab/auth_manager/decorators/test_auth.py index d6f60bf42f1fb..98f77a4f34271 100644 --- a/tests/providers/fab/auth_manager/decorators/test_auth.py +++ b/tests/providers/fab/auth_manager/decorators/test_auth.py @@ -19,9 +19,9 @@ from unittest.mock import Mock, patch import pytest -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.security.permissions import ACTION_CAN_READ, RESOURCE_DAG +from tests.test_utils.compat import ignore_provider_compatibility_error permissions = [(ACTION_CAN_READ, RESOURCE_DAG)] diff --git a/tests/providers/fab/auth_manager/models/test_anonymous_user.py b/tests/providers/fab/auth_manager/models/test_anonymous_user.py index eaf6b357f9264..4e365e3c8b705 100644 --- a/tests/providers/fab/auth_manager/models/test_anonymous_user.py +++ b/tests/providers/fab/auth_manager/models/test_anonymous_user.py @@ -17,7 +17,7 @@ # under the License. from __future__ import annotations -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser diff --git a/tests/providers/fab/auth_manager/models/test_db.py b/tests/providers/fab/auth_manager/models/test_db.py index 3af94ceed7b18..54c10849e966c 100644 --- a/tests/providers/fab/auth_manager/models/test_db.py +++ b/tests/providers/fab/auth_manager/models/test_db.py @@ -25,6 +25,7 @@ from sqlalchemy import MetaData import airflow.providers +from airflow.exceptions import AirflowOptionalProviderFeatureException from airflow.settings import engine from airflow.utils.db import ( compare_server_default, @@ -128,5 +129,5 @@ def test_resetdb(self, mock_initdb, mock_drop_tables, session, skip_init): mock_initdb.assert_not_called() else: mock_initdb.assert_called_once() -except ModuleNotFoundError: +except (ModuleNotFoundError, AirflowOptionalProviderFeatureException): pass diff --git a/tests/providers/fab/auth_manager/schemas/test_role_and_permission_schema.py b/tests/providers/fab/auth_manager/schemas/test_role_and_permission_schema.py index f8364a2e47222..e9e10eb040868 100644 --- a/tests/providers/fab/auth_manager/schemas/test_role_and_permission_schema.py +++ b/tests/providers/fab/auth_manager/schemas/test_role_and_permission_schema.py @@ -17,7 +17,6 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role from airflow.providers.fab.auth_manager.schemas.role_and_permission_schema import ( RoleCollection, @@ -25,6 +24,7 @@ role_schema, ) from airflow.security import permissions +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/providers/fab/auth_manager/schemas/test_user_schema.py b/tests/providers/fab/auth_manager/schemas/test_user_schema.py index f6b07327c09ce..26645e5cdbcbc 100644 --- a/tests/providers/fab/auth_manager/schemas/test_user_schema.py +++ b/tests/providers/fab/auth_manager/schemas/test_user_schema.py @@ -17,10 +17,10 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.utils import timezone +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_role, delete_role +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import User diff --git a/tests/providers/fab/auth_manager/security_manager/test_constants.py b/tests/providers/fab/auth_manager/security_manager/test_constants.py index dbe592c59d747..5a718eee4b639 100644 --- a/tests/providers/fab/auth_manager/security_manager/test_constants.py +++ b/tests/providers/fab/auth_manager/security_manager/test_constants.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.constants import EXISTING_ROLES diff --git a/tests/providers/fab/auth_manager/security_manager/test_override.py b/tests/providers/fab/auth_manager/security_manager/test_override.py index 6ba1ccda292cd..6d85c0319dc44 100644 --- a/tests/providers/fab/auth_manager/security_manager/test_override.py +++ b/tests/providers/fab/auth_manager/security_manager/test_override.py @@ -19,7 +19,7 @@ from unittest import mock from unittest.mock import Mock -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride diff --git a/tests/providers/fab/auth_manager/test_fab_auth_manager.py b/tests/providers/fab/auth_manager/test_fab_auth_manager.py index d298f7667eaaf..1ccd18c44793b 100644 --- a/tests/providers/fab/auth_manager/test_fab_auth_manager.py +++ b/tests/providers/fab/auth_manager/test_fab_auth_manager.py @@ -32,7 +32,7 @@ except ImportError: pass -from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager diff --git a/tests/providers/fab/auth_manager/test_models.py b/tests/providers/fab/auth_manager/test_models.py index 76b69c3dea734..30677d7095753 100644 --- a/tests/providers/fab/auth_manager/test_models.py +++ b/tests/providers/fab/auth_manager/test_models.py @@ -19,7 +19,8 @@ from unittest import mock from sqlalchemy import Column, MetaData, String, Table -from tests_common.test_utils.compat import ignore_provider_compatibility_error + +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.models import ( diff --git a/tests/providers/fab/auth_manager/test_security.py b/tests/providers/fab/auth_manager/test_security.py index eb99daf9b9b53..b0113e3ac614e 100644 --- a/tests/providers/fab/auth_manager/test_security.py +++ b/tests/providers/fab/auth_manager/test_security.py @@ -30,36 +30,35 @@ from flask_appbuilder import SQLA, Model, expose, has_access from flask_appbuilder.views import BaseView, ModelView from sqlalchemy import Column, Date, Float, Integer, String -from tests_common.test_utils.compat import ignore_provider_compatibility_error from airflow.configuration import initialize_config from airflow.exceptions import AirflowException from airflow.models import DagModel from airflow.models.dag import DAG +from tests.test_utils.compat import ignore_provider_compatibility_error with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager from airflow.providers.fab.auth_manager.models import assoc_permission_role from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import ( - create_user, - create_user_scope, - delete_role, - delete_user, - set_user_single_role, -) -from tests_common.test_utils.asserts import assert_queries_count -from tests_common.test_utils.db import clear_db_dags, clear_db_runs -from tests_common.test_utils.mock_security_manager import MockSecurityManager -from tests_common.test_utils.permissions import _resource_name - from airflow.security import permissions from airflow.security.permissions import ACTION_CAN_READ from airflow.www import app as application from airflow.www.auth import get_access_denied_message from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.utils import CustomSQLAInterface +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( + create_user, + create_user_scope, + delete_role, + delete_user, + set_user_single_role, +) +from tests.test_utils.asserts import assert_queries_count +from tests.test_utils.db import clear_db_dags, clear_db_runs +from tests.test_utils.mock_security_manager import MockSecurityManager +from tests.test_utils.permissions import _resource_name pytestmark = pytest.mark.db_test diff --git a/tests/providers/fab/auth_manager/views/test_permissions.py b/tests/providers/fab/auth_manager/views/test_permissions.py index d80ad66e6e366..4b81e814986a0 100644 --- a/tests/providers/fab/auth_manager/views/test_permissions.py +++ b/tests/providers/fab/auth_manager/views/test_permissions.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from providers.tests.fab.auth_manager.views import _assert_dataset_deprecation_warning -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.providers.fab.auth_manager.views import _assert_dataset_deprecation_warning +from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_roles_list.py b/tests/providers/fab/auth_manager/views/test_roles_list.py index 79b11b55fa5b1..94e6677fdecb9 100644 --- a/tests/providers/fab/auth_manager/views/test_roles_list.py +++ b/tests/providers/fab/auth_manager/views/test_roles_list.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from providers.tests.fab.auth_manager.views import _assert_dataset_deprecation_warning -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.providers.fab.auth_manager.views import _assert_dataset_deprecation_warning +from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_user.py b/tests/providers/fab/auth_manager/views/test_user.py index 1f33d14cac2d0..4ccd61b100ce5 100644 --- a/tests/providers/fab/auth_manager/views/test_user.py +++ b/tests/providers/fab/auth_manager/views/test_user.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from providers.tests.fab.auth_manager.views import _assert_dataset_deprecation_warning -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.providers.fab.auth_manager.views import _assert_dataset_deprecation_warning +from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_user_edit.py b/tests/providers/fab/auth_manager/views/test_user_edit.py index 7279ed0b2c601..37f313fde1fca 100644 --- a/tests/providers/fab/auth_manager/views/test_user_edit.py +++ b/tests/providers/fab/auth_manager/views/test_user_edit.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from providers.tests.fab.auth_manager.views import _assert_dataset_deprecation_warning -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.providers.fab.auth_manager.views import _assert_dataset_deprecation_warning +from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/fab/auth_manager/views/test_user_stats.py b/tests/providers/fab/auth_manager/views/test_user_stats.py index 382a8f10984ac..ee818589fa5eb 100644 --- a/tests/providers/fab/auth_manager/views/test_user_stats.py +++ b/tests/providers/fab/auth_manager/views/test_user_stats.py @@ -18,13 +18,13 @@ from __future__ import annotations import pytest -from providers.tests.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user -from providers.tests.fab.auth_manager.views import _assert_dataset_deprecation_warning -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS -from tests_common.test_utils.www import client_with_login from airflow.security import permissions from airflow.www import app as application +from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import create_user, delete_user +from tests.providers.fab.auth_manager.views import _assert_dataset_deprecation_warning +from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests.test_utils.www import client_with_login pytestmark = [ pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"), diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index 962429e30eb9e..96b5b6e7d2af2 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -26,7 +26,7 @@ import pytest from attrs import define from openlineage.client.utils import RedactMixin -from pkg_resources import parse_version +from packaging.version import parse as parse_version from airflow.models import DAG as AIRFLOW_DAG, DagModel from airflow.operators.bash import BashOperator diff --git a/tests/test_utils/compat.py b/tests/test_utils/compat.py index b5e876a626add..fc3e492760f22 100644 --- a/tests/test_utils/compat.py +++ b/tests/test_utils/compat.py @@ -46,6 +46,7 @@ AIRFLOW_V_2_8_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("2.8.0") AIRFLOW_V_2_9_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("2.9.0") AIRFLOW_V_2_10_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("2.10.0") +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") try: from airflow.models.baseoperatorlink import BaseOperatorLink diff --git a/tests/test_utils/www.py b/tests/test_utils/www.py index 0a19c312fba4e..6d105384efd40 100644 --- a/tests/test_utils/www.py +++ b/tests/test_utils/www.py @@ -62,9 +62,9 @@ def check_content_not_in_response(text, resp, resp_code=200): assert resp_code == resp.status_code if isinstance(text, list): for line in text: - assert line not in resp_html + assert line not in resp_html, f"Found {line!r} but it shouldn't be there" else: - assert text not in resp_html + assert text not in resp_html, f"Found {text!r} but it shouldn't be there" def _check_last_log(session, dag_id, event, execution_date, expected_extra=None): diff --git a/tests/www/views/conftest.py b/tests/www/views/conftest.py index 821f541ef0c43..6f5e70ce7dd74 100644 --- a/tests/www/views/conftest.py +++ b/tests/www/views/conftest.py @@ -35,7 +35,7 @@ @pytest.fixture(autouse=True, scope="module") def session(): - settings.configure_orm() + settings.reconfigure_orm() return settings.Session diff --git a/tests/www/views/test_session.py b/tests/www/views/test_session.py index 8a69d4b3525ea..45c4d3152a4ed 100644 --- a/tests/www/views/test_session.py +++ b/tests/www/views/test_session.py @@ -66,12 +66,13 @@ def poorly_configured_app_factory(): with conf_vars({("webserver", "session_backend"): "invalid_value_for_session_backend"}): return app.create_app(testing=True) - expected_exc_regex = ( - "^Unrecognized session backend specified in web_server_session_backend: " - r"'invalid_value_for_session_backend'\. Please set this to .+\.$" - ) - with pytest.raises(AirflowConfigException, match=expected_exc_regex): - poorly_configured_app_factory() + with conf_vars({("fab", "auth_rate_limited"): "False"}): + expected_exc_regex = ( + "^Unrecognized session backend specified in web_server_session_backend: " + r"'invalid_value_for_session_backend'\. Please set this to .+\.$" + ) + with pytest.raises(AirflowConfigException, match=expected_exc_regex): + poorly_configured_app_factory() def test_session_id_rotates(app, user_client): diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index 01d1bb731e635..5452dc7234242 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -304,7 +304,8 @@ def test_get_safe_url(mock_url_for, app, test_url, expected_url): def test_app(): from airflow.www import app - return app.create_app(testing=True) + with conf_vars({("fab", "auth_rate_limited"): "True"}): + return app.create_app(testing=True) def test_mark_task_instance_state(test_app): diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index 9586d81aa6516..cb40765773251 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -26,7 +26,6 @@ from airflow.models import DagModel from airflow.security import permissions from airflow.utils import timezone -from airflow.utils.session import create_session from airflow.utils.state import State from airflow.utils.types import DagRunType from airflow.www.views import FILTER_STATUS_COOKIE @@ -315,14 +314,13 @@ def test_dag_autocomplete_dag_display_name(client_all_dags): @pytest.fixture -def setup_paused_dag(): +def setup_paused_dag(app): """Pause a DAG so we can test filtering.""" + session = app.appbuilder.get_session dag_to_pause = "example_branch_operator" - with create_session() as session: - session.query(DagModel).filter(DagModel.dag_id == dag_to_pause).update({"is_paused": True}) + session.query(DagModel).filter(DagModel.dag_id == dag_to_pause).update({"is_paused": True}) yield - with create_session() as session: - session.query(DagModel).filter(DagModel.dag_id == dag_to_pause).update({"is_paused": False}) + session.query(DagModel).filter(DagModel.dag_id == dag_to_pause).update({"is_paused": False}) @pytest.mark.parametrize( diff --git a/tests/www/views/test_views_base.py b/tests/www/views/test_views_base.py index a125ca2d72835..1139982945a6a 100644 --- a/tests/www/views/test_views_base.py +++ b/tests/www/views/test_views_base.py @@ -407,6 +407,7 @@ def test_page_instance_name_xss_prevention(admin_client): instance_name_with_markup_conf = { ("webserver", "instance_name"): "Bold Site Title Test", ("webserver", "instance_name_has_markup"): "True", + ("fab", "auth_rate_limited"): "True", } diff --git a/tests/www/views/test_views_custom_user_views.py b/tests/www/views/test_views_custom_user_views.py index 8182d3ebefa82..c04124a9af134 100644 --- a/tests/www/views/test_views_custom_user_views.py +++ b/tests/www/views/test_views_custom_user_views.py @@ -166,7 +166,7 @@ def test_user_model_view_with_delete_access(self): ) response = client.post(f"/users/delete/{user_to_delete.id}", follow_redirects=True) - check_content_in_response("Deleted Row", response) + check_content_in_response("User confirmation needed", response) check_content_not_in_response(user_to_delete.username, response) assert bool(self.security_manager.get_user_by_id(user_to_delete.id)) is False diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 79dd4a57feef7..b55cf41802bf5 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -265,86 +265,110 @@ def test_rendered_template_secret(admin_client, create_dag_run, task_secret): @pytest.mark.enable_redact @pytest.mark.parametrize( - "env, expected", + "env_spec, expected, var_setup", [ pytest.param( {"plain_key": "plain_value"}, "{'plain_key': 'plain_value'}", + None, id="env-plain-key-val", ), pytest.param( - {"plain_key": Variable.setdefault("plain_var", "banana")}, + {"plain_key": ("var", "plain_var", "banana")}, "{'plain_key': 'banana'}", + {"plain_var": "banana"}, id="env-plain-key-plain-var", ), pytest.param( - {"plain_key": Variable.setdefault("secret_var", "monkey")}, + {"plain_key": ("var", "secret_var", "monkey")}, "{'plain_key': '***'}", + {"secret_var": "monkey"}, id="env-plain-key-sensitive-var", ), pytest.param( {"plain_key": "{{ var.value.plain_var }}"}, "{'plain_key': '{{ var.value.plain_var }}'}", + {"plain_var": "banana"}, id="env-plain-key-plain-tpld-var", ), pytest.param( {"plain_key": "{{ var.value.secret_var }}"}, "{'plain_key': '{{ var.value.secret_var }}'}", + {"secret_var": "monkey"}, id="env-plain-key-sensitive-tpld-var", ), pytest.param( {"secret_key": "plain_value"}, "{'secret_key': '***'}", + None, id="env-sensitive-key-plain-val", ), pytest.param( - {"secret_key": Variable.setdefault("plain_var", "monkey")}, + {"secret_key": ("var", "plain_var", "monkey")}, "{'secret_key': '***'}", + {"plain_var": "monkey"}, id="env-sensitive-key-plain-var", ), pytest.param( - {"secret_key": Variable.setdefault("secret_var", "monkey")}, + {"secret_key": ("var", "secret_var", "monkey")}, "{'secret_key': '***'}", + {"secret_var": "monkey"}, id="env-sensitive-key-sensitive-var", ), pytest.param( {"secret_key": "{{ var.value.plain_var }}"}, "{'secret_key': '***'}", + {"plain_var": "banana"}, id="env-sensitive-key-plain-tpld-var", ), pytest.param( {"secret_key": "{{ var.value.secret_var }}"}, "{'secret_key': '***'}", + {"secret_var": "monkey"}, id="env-sensitive-key-sensitive-tpld-var", ), ], ) -def test_rendered_task_detail_env_secret(patch_app, admin_client, request, env, expected): - if request.node.callspec.id.endswith("-tpld-var"): - Variable.set("plain_var", "banana") - Variable.set("secret_var", "monkey") - - dag: DAG = patch_app.dag_bag.get_dag("testdag") - task_secret: BashOperator = dag.get_task(task_id="task1") - task_secret.env = env - date = quote_plus(str(DEFAULT_DATE)) - url = f"task?task_id=task1&dag_id=testdag&execution_date={date}" - - with create_session() as session: - dag.create_dagrun( - state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - run_type=DagRunType.SCHEDULED, - session=session, - ) - - resp = admin_client.get(url, follow_redirects=True) - check_content_in_response(str(escape(expected)), resp) - - if request.node.callspec.id.endswith("-tpld-var"): - Variable.delete("plain_var") - Variable.delete("secret_var") +def test_rendered_task_detail_env_secret(patch_app, admin_client, env_spec, expected, var_setup): + # Setup variables if needed + if var_setup: + for var_name, var_value in var_setup.items(): + Variable.set(var_name, var_value) + + # Build the actual env dict from spec + env = {} + for key, value in env_spec.items(): + if isinstance(value, tuple) and value[0] == "var": + # This is a variable reference - call setdefault now + _, var_name, default_value = value + env[key] = Variable.setdefault(var_name, default_value) + else: + # Plain value or template string + env[key] = value + + try: + dag: DAG = patch_app.dag_bag.get_dag("testdag") + task_secret: BashOperator = dag.get_task(task_id="task1") + task_secret.env = env + date = quote_plus(str(DEFAULT_DATE)) + url = f"task?task_id=task1&dag_id=testdag&execution_date={date}" + + with create_session() as session: + dag.create_dagrun( + state=DagRunState.RUNNING, + execution_date=DEFAULT_DATE, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + run_type=DagRunType.SCHEDULED, + session=session, + ) + + resp = admin_client.get(url, follow_redirects=True) + check_content_in_response(str(escape(expected)), resp) + finally: + # Cleanup variables + if var_setup: + for var_name in var_setup.keys(): + Variable.delete(var_name) @pytest.mark.usefixtures("patch_app")