diff --git a/.dockerignore b/.dockerignore index edce6e9e78aef..c7fb2e60c7bb1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -47,6 +47,7 @@ !helm-tests !kubernetes-tests !task-sdk-tests +!airflow-ctl-tests !shared/ # Add scripts so that we can use them inside the container diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 714761bceb3cc..c7d4bde61c8db 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -191,3 +191,34 @@ jobs: id: breeze - name: "Run Task SDK integration tests" run: breeze testing task-sdk-integration-tests + + airflow-ctl-integration-tests: + timeout-minutes: 60 + name: "Airflow CTL integration tests with PROD image" + runs-on: ${{ fromJSON(inputs.runners) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + image-type: "prod" + python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + use-uv: ${{ inputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Run airflowctl integration tests" + run: breeze testing airflow-ctl-integration-tests diff --git a/Dockerfile b/Dockerfile index 0f7f5007b3a35..8fab2a4ae015c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1116,6 +1116,7 @@ function install_from_sources() { --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \ --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \ --editable ./task-sdk-tests \ + --editable ./airflow-ctl-tests \ --editable ./devel-common[all] --editable ./dev \ --group dev --group docs --group docs-gen --group leveldb" local -a projects_with_devel_dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 74c7692031d8d..744fe240a10aa 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -870,6 +870,7 @@ function install_from_sources() { --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \ --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \ --editable ./task-sdk-tests \ + --editable ./airflow-ctl-tests \ --editable ./devel-common[all] --editable ./dev \ --group dev --group docs --group docs-gen --group leveldb" local -a projects_with_devel_dependencies diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index 2c2a614c9ef72..3892e04414358 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -51,6 +51,8 @@ x-airflow-common: # and uncomment the "build" line below, Then run `docker-compose build` to build the images. image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:|version|} # build: . + env_file: + - ${ENV_FILE_PATH:-.env} environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor diff --git a/airflow-ctl-tests/pyproject.toml b/airflow-ctl-tests/pyproject.toml new file mode 100644 index 0000000000000..d401975ad73f4 --- /dev/null +++ b/airflow-ctl-tests/pyproject.toml @@ -0,0 +1,64 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = [ "hatchling==1.27.0" ] +build-backend = "hatchling.build" + +[project] +name = "apache-airflow-ctl-tests" +description = "Airflow CTL tests for Apache Airflow" +classifiers = [ + "Private :: Do Not Upload", +] +requires-python = ">=3.10,!=3.13" +authors = [ + { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, +] +maintainers = [ + { name = "Apache Software Foundation", email="dev@airflow.apache.org" }, +] +version = "0.0.1" + +dependencies = [ + "apache-airflow-ctl", + "apache-airflow-devel-common", +] + +[tool.pytest.ini_options] +addopts = "-rasl --verbosity=2 -p no:flaky -p no:nose -p no:legacypath" +norecursedirs = [ + ".eggs", +] +log_level = "INFO" +filterwarnings = [ + "error::pytest.PytestCollectionWarning", +] +python_files = [ + "*.py", +] + +# Keep temporary directories (created by `tmp_path`) for 2 recent runs only failed tests. +tmp_path_retention_count = "2" +tmp_path_retention_policy = "failed" + +[tool.hatch.build.targets.sdist] +exclude = ["*"] + +[tool.hatch.build.targets.wheel] +bypass-selection = true diff --git a/airflow-ctl-tests/tests/airflowctl_tests/__init__.py b/airflow-ctl-tests/tests/airflowctl_tests/__init__.py new file mode 100644 index 0000000000000..973b8fdce34e8 --- /dev/null +++ b/airflow-ctl-tests/tests/airflowctl_tests/__init__.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from rich.console import Console + +console = Console(width=400, color_system="standard") + + +__all__ = ["console"] diff --git a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py new file mode 100644 index 0000000000000..202f5d3ca4e83 --- /dev/null +++ b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py @@ -0,0 +1,253 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import subprocess +import sys + +import pytest +from python_on_whales import DockerClient, docker + +from airflowctl_tests import console +from airflowctl_tests.constants import ( + AIRFLOW_ROOT_PATH, + DOCKER_COMPOSE_FILE_PATH, + DOCKER_IMAGE, +) + +docker_client = None + + +# Pytest hook to run at the start of the session +def pytest_sessionstart(session): + """Install airflowctl at the very start of the pytest session.""" + airflow_ctl_version = os.environ.get("AIRFLOW_CTL_VERSION", "1.0.0") + console.print(f"[yellow]Installing apache-airflow-ctl=={airflow_ctl_version} via pytest_sessionstart...") + + airflow_ctl_path = AIRFLOW_ROOT_PATH / "airflow-ctl" + console.print(f"[blue]Installing from: {airflow_ctl_path}") + + # Install directly to current UV environment + console.print("[blue]Installing to current UV environment...") + console.print(f"[blue]Current Python: {sys.executable}") + + try: + cmd = ["uv", "pip", "install", str(airflow_ctl_path)] + console.print(f"[cyan]Running command: {' '.join(cmd)}") + subprocess.check_call(cmd) + console.print("[green]airflowctl installed successfully to UV environment via pytest_sessionstart!") + except (subprocess.CalledProcessError, FileNotFoundError) as e: + console.print(f"[yellow]UV installation failed: {e}") + raise + + console.print("[yellow]Verifying airflowctl installation via pytest_sessionstart...") + try: + result = subprocess.run( + [ + sys.executable, + "-c", + "import airflowctl.api.client; print('✅ airflowctl import successful via pytest_sessionstart')", + ], + capture_output=True, + text=True, + check=True, + ) + console.print(f"[green]{result.stdout.strip()}") + except subprocess.CalledProcessError as e: + console.print("[red]❌ airflowctl import verification failed via pytest_sessionstart:") + console.print(f"[red]Return code: {e.returncode}") + console.print(f"[red]Stdout: {e.stdout}") + console.print(f"[red]Stderr: {e.stderr}") + raise + + docker_compose_up(session.config._tmp_path_factory) + + +def print_diagnostics(compose, compose_version, docker_version): + """Print diagnostic information when test fails.""" + console.print("[red]=== DIAGNOSTIC INFORMATION ===[/]") + console.print(f"Docker version: {docker_version}") + console.print(f"Docker Compose version: {compose_version}") + console.print("\n[yellow]Container Status:[/]") + try: + containers = compose.compose.ps() + for container in containers: + console.print(f" {container.name}: {container.state}") + except Exception as e: + console.print(f" Error getting container status: {e}") + + console.print("\n[yellow]Container Logs:[/]") + try: + logs = compose.compose.logs() + console.print(logs) + except Exception as e: + console.print(f" Error getting logs: {e}") + + +def debug_environment(): + """Debug the Python environment setup in CI.""" + import os + import subprocess + import sys + from pathlib import Path + + console.print("[yellow]===== CI ENVIRONMENT DEBUG =====") + console.print(f"[blue]Python executable: {sys.executable}") + console.print(f"[blue]Python version: {sys.version}") + console.print(f"[blue]Working directory: {os.getcwd()}") + console.print(f"[blue]VIRTUAL_ENV: {os.environ.get('VIRTUAL_ENV', 'Not set')}") + console.print(f"[blue]PYTHONPATH: {os.environ.get('PYTHONPATH', 'Not set')}") + + console.print(f"[blue]Python executable exists: {Path(sys.executable).exists()}") + if Path(sys.executable).is_symlink(): + console.print(f"[blue]Python executable is symlink to: {Path(sys.executable).readlink()}") + + try: + uv_python = subprocess.check_output(["uv", "python", "find"], text=True).strip() + console.print(f"[cyan]UV Python: {uv_python}") + console.print(f"[green]Match: {uv_python == sys.executable}") + + console.print(f"[cyan]UV Python exists: {Path(uv_python).exists()}") + if Path(uv_python).is_symlink(): + console.print(f"[cyan]UV Python is symlink to: {Path(uv_python).readlink()}") + except Exception as e: + console.print(f"[red]UV Python error: {e}") + + # Check what's installed in current environment + try: + import airflowctl + + console.print(f"[green]✅ airflow already available: {airflowctl.__file__}") + except ImportError: + console.print("[red]❌ airflowctl not available in current environment") + + console.print("[yellow]================================") + + +def docker_compose_up(tmp_path_factory): + """Fixture to spin up Docker Compose environment for the test session.""" + from shutil import copyfile + + global docker_client + + tmp_dir = tmp_path_factory.mktemp("airflow-ctl-test") + console.print(f"[yellow]Tests are run in {tmp_dir}") + + # Copy docker-compose.yaml to temp directory + tmp_docker_compose_file = tmp_dir / "docker-compose.yaml" + copyfile(DOCKER_COMPOSE_FILE_PATH, tmp_docker_compose_file) + + dot_env_file = tmp_dir / ".env" + dot_env_file.write_text( + f"AIRFLOW_UID={os.getuid()}\n" + # To enable debug mode for airflowctl CLI + "AIRFLOW_CTL_CLI_DEBUG_MODE=true\n" + # To enable config operations to work + "AIRFLOW__API__EXPOSE_CONFIG=true\n" + ) + + # Set environment variables for the test + os.environ["AIRFLOW_IMAGE_NAME"] = DOCKER_IMAGE + os.environ["AIRFLOW_CTL_VERSION"] = os.environ.get("AIRFLOW_CTL_VERSION", "1.0.0") + os.environ["ENV_FILE_PATH"] = str(tmp_dir / ".env") + + # Initialize Docker client + docker_client = DockerClient(compose_files=[str(tmp_docker_compose_file)]) + + try: + console.print(f"[blue]Spinning up airflow environment using {DOCKER_IMAGE}") + docker_client.compose.up(detach=True, wait=True) + console.print("[green]Docker compose started for airflowctl test\n") + except Exception: + print_diagnostics(docker_client.compose, docker_client.compose.version(), docker.version()) + debug_environment() + docker_compose_down() + raise + + +def docker_compose_down(): + """Tear down Docker Compose environment.""" + global docker_client + if docker_client: + docker_client.compose.down(remove_orphans=True, volumes=True, quiet=True) + + +def pytest_sessionfinish(session, exitstatus): + """Tear down test environment at the end of the pytest session.""" + if not os.environ.get("SKIP_DOCKER_COMPOSE_DELETION"): + docker_compose_down() + + +# Fixtures for tests +@pytest.fixture +def login_command(): + # Passing password via command line is insecure but acceptable for testing purposes + # Please do not do this in production, it enables possibility of exposing your credentials + return "auth login --username airflow --password airflow" + + +@pytest.fixture +def login_output(): + return "Login successful! Welcome to airflowctl!" + + +@pytest.fixture +def date_param(): + import random + from datetime import datetime, timedelta + + from dateutil.relativedelta import relativedelta + + # original datetime string + dt_str = "2025-10-25T00:02:00+00:00" + + # parse to datetime object + dt = datetime.fromisoformat(dt_str) + + # boundaries + start = dt - relativedelta(months=1) + end = dt + relativedelta(months=1) + + # pick random time between start and end + delta = end - start + random_seconds = random.randint(0, int(delta.total_seconds())) + random_dt = start + timedelta(seconds=random_seconds) + return random_dt.isoformat() + + +@pytest.fixture +def test_commands(login_command, date_param): + # Define test commands to run with actual running API server + return [ + login_command, + "backfills list", + "config get --section core --option executor", + "connections create --connection-id=test_con --conn-type=mysql --password=TEST_PASS -o json", + "connections list", + "connections list -o yaml", + "connections list -o tabledags list", + f"dagrun trigger --dag-id=example_bash_operator --logical-date={date_param} --run-after={date_param}", + "dagrun list --dag-id example_bash_operator --state success --limit=1", + "jobs list", + "pools create --name=test_pool --slots=5", + "pools list", + "providers list", + "variables create --key=test_key --value=test_value", + "variables list", + "version --remote", + ] diff --git a/airflow-ctl-tests/tests/airflowctl_tests/constants.py b/airflow-ctl-tests/tests/airflowctl_tests/constants.py new file mode 100644 index 0000000000000..549a9ad8c7439 --- /dev/null +++ b/airflow-ctl-tests/tests/airflowctl_tests/constants.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from pathlib import Path + +AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3] + +DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10" +DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest" +DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE + +DOCKER_COMPOSE_HOST_PORT = os.environ.get("HOST_PORT", "localhost:8080") + +DOCKER_COMPOSE_FILE_PATH = ( + AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "howto" / "docker-compose" / "docker-compose.yaml" +) diff --git a/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py b/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py new file mode 100644 index 0000000000000..d17e646cb069d --- /dev/null +++ b/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from subprocess import PIPE, STDOUT, Popen + +from airflowctl_tests import console + + +def test_airflowctl_commands(login_command, login_output, test_commands): + """Test airflowctl commands using docker-compose environment.""" + host_envs = os.environ.copy() + host_envs["AIRFLOW_CLI_DEBUG_MODE"] = "true" + # Testing commands of airflowctl + for command in test_commands: + command_from_config = f"airflowctl {command}" + # We need to run auth login first for all commands except login itself + if command != login_command: + run_command = f"airflowctl {login_command} && {command_from_config}" + else: + run_command = command_from_config + console.print(f"[yellow]Running command: {command}") + + # Give some time for the command to execute and output to be ready + proc = Popen(run_command.encode(), stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True, env=host_envs) + stdout_result, stderr_result = proc.communicate(timeout=60) + + # CLI command gave errors + if stderr_result: + console.print( + f"[red]Errors while executing command '{command_from_config}':\n{stderr_result.decode()}" + ) + + # Decode the output + stdout_result = stdout_result.decode() + # We need to trim auth login output if the command is not login itself and clean backspaces + if command != login_command: + if login_output not in stdout_result: + console.print( + f"[red]❌ Login output not found before command output for '{command_from_config}'" + ) + console.print(f"[red]Full output:\n{stdout_result}\n") + raise AssertionError("Login output not found before command output") + stdout_result = stdout_result.split(f"{login_output}\n")[1].strip() + else: + stdout_result = stdout_result.strip() + + # This is a common error message that is thrown by client when something is wrong + # Please ensure it is aligning with airflowctl.api.client.get_json_error + airflowctl_client_server_response_error = "Server error" + airflowctl_command_error = "command error: argument GROUP_OR_COMMAND: invalid choice" + if ( + airflowctl_client_server_response_error in stdout_result + or airflowctl_command_error in stdout_result + ): + console.print(f"[red]❌ Output contained unexpected text for command '{command_from_config}'") + console.print(f"[red]Did not expect to find:\n{airflowctl_client_server_response_error}\n") + console.print(f"[red]But got:\n{stdout_result}\n") + raise AssertionError(f"Output contained unexpected text\nOutput:\n{stdout_result}") + console.print(f"[green]✅ Output did not contain unexpected text for command '{command_from_config}'") + console.print(f"[cyan]Result:\n{stdout_result}\n") + proc.kill() diff --git a/airflow-ctl/docs/cli-and-env-variables-ref.rst b/airflow-ctl/docs/cli-and-env-variables-ref.rst index aebe2c565792c..2d63d47d5be53 100644 --- a/airflow-ctl/docs/cli-and-env-variables-ref.rst +++ b/airflow-ctl/docs/cli-and-env-variables-ref.rst @@ -53,3 +53,10 @@ Environment Variables required if you have multiple environments set up and want to specify which one to use. If not set, the default environment will be used which is production. + +.. envvar:: AIRFLOW_CLI_DEBUG_MODE + + This variable can be used to enable debug mode for the CLI. + It disables some features such as keyring integration and save credentials to file. + It is only meant to use if either you are developing airflowctl or running API integration tests. + Please do not use this variable unless you know what you are doing. diff --git a/airflow-ctl/src/airflowctl/api/client.py b/airflow-ctl/src/airflowctl/api/client.py index 4f2ee2fbd8d14..09947833bafa3 100644 --- a/airflow-ctl/src/airflowctl/api/client.py +++ b/airflow-ctl/src/airflowctl/api/client.py @@ -94,6 +94,8 @@ def get_json_error(response: httpx.Response): """Raise a ServerResponseError if we can extract error info from the error.""" err = ServerResponseError.from_response(response) if err: + # This part is used in integration tests to verify the error message + # If you are updating here don't forget to update the airflow-ctl-tests log.warning("Server error ", extra=dict(err.response.json())) raise err @@ -133,8 +135,15 @@ def save(self): os.makedirs(default_config_dir, exist_ok=True) with open(os.path.join(default_config_dir, self.input_cli_config_file), "w") as f: json.dump({"api_url": self.api_url}, f) + try: - keyring.set_password("airflowctl", f"api_token-{self.api_environment}", self.api_token) + if os.getenv("AIRFLOW_CLI_DEBUG_MODE") == "true": + with open( + os.path.join(default_config_dir, f"debug_creds_{self.input_cli_config_file}"), "w" + ) as f: + json.dump({f"api_token_{self.api_environment}": self.api_token}, f) + else: + keyring.set_password("airflowctl", f"api_token_{self.api_environment}", self.api_token) except NoKeyringError as e: log.error(e) except TypeError as e: @@ -145,12 +154,20 @@ def save(self): def load(self) -> Credentials: """Load the credentials from keyring and URL from disk file.""" default_config_dir = os.environ.get("AIRFLOW_HOME", os.path.expanduser("~/airflow")) - credential_path = os.path.join(default_config_dir, self.input_cli_config_file) + config_path = os.path.join(default_config_dir, self.input_cli_config_file) try: - with open(credential_path) as f: + with open(config_path) as f: credentials = json.load(f) self.api_url = credentials["api_url"] - self.api_token = keyring.get_password("airflowctl", f"api_token-{self.api_environment}") + if os.getenv("AIRFLOW_CLI_DEBUG_MODE") == "true": + debug_creds_path = os.path.join( + default_config_dir, f"debug_creds_{self.input_cli_config_file}" + ) + with open(debug_creds_path) as df: + debug_credentials = json.load(df) + self.api_token = debug_credentials.get(f"api_token_{self.api_environment}") + else: + self.api_token = keyring.get_password("airflowctl", f"api_token_{self.api_environment}") except FileNotFoundError: if self.client_kind == ClientKind.AUTH: # Saving the URL set from the Auth Commands if Kind is AUTH diff --git a/airflow-ctl/src/airflowctl/ctl/cli_config.py b/airflow-ctl/src/airflowctl/ctl/cli_config.py index 085fe5abec473..fb6a7dfde0d24 100644 --- a/airflow-ctl/src/airflowctl/ctl/cli_config.py +++ b/airflow-ctl/src/airflowctl/ctl/cli_config.py @@ -66,6 +66,12 @@ def command(*args, **kwargs): def safe_call_command(function: Callable, args: Iterable[Arg]) -> None: import sys + if os.getenv("AIRFLOW_CLI_DEBUG_MODE") == "true": + rich.print( + "[yellow]Debug mode is enabled. Please be aware that your credentials are not secure.\n" + "Please unset AIRFLOW_CLI_DEBUG_MODE or set it to false.[/yellow]" + ) + try: function(args) except AirflowCtlCredentialNotFoundException as e: @@ -187,7 +193,8 @@ class Password(argparse.Action): """Custom action to prompt for password input.""" def __call__(self, parser, namespace, values, option_string=None): - values = getpass.getpass() + if values is None: + values = getpass.getpass() setattr(namespace, self.dest, values) diff --git a/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py b/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py index c5285710d58c3..0ad428ee5baee 100644 --- a/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py +++ b/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py @@ -67,7 +67,7 @@ def test_login(self, mock_keyring, api_client_maker, monkeypatch): assert json.load(f) == {"api_url": "http://localhost:8080"} mock_keyring.set_password.assert_called_once_with( - "airflowctl", "api_token-TEST_AUTH_LOGIN", "TEST_TOKEN" + "airflowctl", "api_token_TEST_AUTH_LOGIN", "TEST_TOKEN" ) # Test auth login with username and password @@ -102,7 +102,7 @@ def test_login_with_username_and_password(self, mock_keyring, api_client_maker): ) mock_keyring.set_password.assert_has_calls( [ - mock.call("airflowctl", "api_token-production", ""), - mock.call("airflowctl", "api_token-production", "TEST_TOKEN"), + mock.call("airflowctl", "api_token_production", ""), + mock.call("airflowctl", "api_token_production", "TEST_TOKEN"), ] ) diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index adb20969803c4..6d46779f74948 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -197,6 +197,25 @@ Here is the detailed set of options for the ``breeze testing airflow-ctl-tests`` :width: 100% :alt: Breeze testing airflow-ctl-tests +Running airflowctl integration tests +.................................. + +You can use Breeze to run the airflowctl integration tests. Those tests are run using Production image by default +and the tests are running with the docker-compose we have for airflowctl tests. + +.. image:: ./images/output_testing_airflow-ctl-integration-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_airflow-ctl-integration-tests.svg + :width: 100% + :alt: Breeze testing airflow-ctl-integration-tests + +You can also iterate over those tests with pytest command but unlike regular unit tests, they need to be run in +a local venv. You can build the prod image with breeze and that will be used by default if present to run the tests. + +You can override the ``DOCKER_IMAGE`` environment variable to point to the image to test using the +``breeze testing airflow-ctl-integration-tests`` command. + +The airflowctl tests are in ``airflow-ctl-tests/`` folder in the main repo. + Running integration core tests ............................... @@ -323,7 +342,7 @@ Running task-sdk integration tests You can use Breeze to run the task sdk integration tests. Those tests are run using Production image by default and the tests are running with the docker-compose we have for task-sdk tests. -.. image:: ./images/output_testing_docker-compose-tests.svg +.. image:: ./images/output_testing_task-sdk-integration-tests.svg :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_task-sdk-integration-tests.svg :width: 100% :alt: Breeze testing task-sdk-integration-tests diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 53d05594312ca..966d335728c44 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -219,11 +219,11 @@ │sbom:export-dependency-information | sbom:generate-providers-requirements |                            │ │sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |         │ │setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts | │ -│setup:version | shell | start-airflow | testing | testing:airflow-ctl-tests |                          │ -│testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests | testing:helm-tests│ -│| testing:providers-integration-tests | testing:providers-tests | testing:python-api-client-tests |    │ -│testing:system-tests | testing:task-sdk-integration-tests | testing:task-sdk-tests | workflow-run |    │ -│workflow-run:publish-docs)                                                                             │ +│setup:version | shell | start-airflow | testing | testing:airflow-ctl-integration-tests |              │ +│testing:airflow-ctl-tests | testing:core-integration-tests | testing:core-tests |                      │ +│testing:docker-compose-tests | testing:helm-tests | testing:providers-integration-tests |              │ +│testing:providers-tests | testing:python-api-client-tests | testing:system-tests |                     │ +│testing:task-sdk-integration-tests | testing:task-sdk-tests | workflow-run | workflow-run:publish-docs)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--verbose-vPrint verbose information about performed steps.│ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt index 2a2e1152f407f..73a88df7b0009 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt @@ -1 +1 @@ -b1fdbd0feeb3b5ba14f79cc8f13d8e1a +d5bec5c837a57060f11d308e952d5e0a diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index bb856ad3a341e..9c53cb893c6da 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -1,4 +1,4 @@ - + │sbom:export-dependency-information | sbom:generate-providers-requirements |                         │ │sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |      │ │setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts│ -│| setup:version | shell | start-airflow | testing | testing:airflow-ctl-tests |                     │ -│testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests |                │ -│testing:helm-tests | testing:providers-integration-tests | testing:providers-tests |                │ -│testing:python-api-client-tests | testing:system-tests | testing:task-sdk-integration-tests |       │ -│testing:task-sdk-tests | workflow-run | workflow-run:publish-docs)                                  │ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ -│together with --command flag or --force.                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│| setup:version | shell | start-airflow | testing | testing:airflow-ctl-integration-tests |         │ +│testing:airflow-ctl-tests | testing:core-integration-tests | testing:core-tests |                   │ +│testing:docker-compose-tests | testing:helm-tests | testing:providers-integration-tests |           │ +│testing:providers-tests | testing:python-api-client-tests | testing:system-tests |                  │ +│testing:task-sdk-integration-tests | testing:task-sdk-tests | workflow-run |                        │ +│workflow-run:publish-docs)                                                                          │ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ +│together with --command flag or --force.                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index f0afa5ca432b1..e6b55de99129c 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -3cb28adadcff7f43ae20d44507368acc +4929732b82fa439a5365a1d3cf5015e3 diff --git a/dev/breeze/doc/images/output_testing.svg b/dev/breeze/doc/images/output_testing.svg index dee85765d4ae9..46f24ee72536f 100644 --- a/dev/breeze/doc/images/output_testing.svg +++ b/dev/breeze/doc/images/output_testing.svg @@ -1,4 +1,4 @@ - +