From c5f7a7d9898c5ede82f6e28f4a956e0e7b5e3e34 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Fri, 1 Dec 2023 12:25:55 -0800 Subject: [PATCH 01/10] expose env vars and dbt vars in ProjectConfig --- .pre-commit-config.yaml | 2 +- cosmos/config.py | 19 +++- cosmos/converter.py | 34 +++++++- cosmos/dbt/graph.py | 18 +++- cosmos/dbt/parser/project.py | 15 ++-- dev/dags/dbt/simple/models/top_animations.sql | 6 +- dev/dags/example_cosmos_sources.py | 15 ++-- docs/configuration/operator-args.rst | 7 -- docs/configuration/project-config.rst | 18 +++- docs/configuration/render-config.rst | 1 - tests/dbt/parser/test_project.py | 2 +- tests/dbt/test_graph.py | 86 +++++++++++++++++++ tests/test_config.py | 6 ++ tests/test_converter.py | 61 ++++++++++++- 14 files changed, 256 insertions(+), 34 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 890afb0b7..8cc2a8b31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,7 +74,7 @@ repos: hooks: - id: mypy name: mypy-python - additional_dependencies: [types-PyYAML, types-attrs, attrs, types-requests, types-python-dateutil, apache-airflow] + additional_dependencies: [types-PyYAML, types-attrs, attrs, types-requests, types-python-dateutil, apache-airflow, jinja2>=3.0.0] files: ^cosmos - repo: https://github.com/pycqa/flake8 rev: 6.1.0 diff --git a/cosmos/config.py b/cosmos/config.py index 40756d2bb..05549afe7 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -7,6 +7,7 @@ import tempfile from dataclasses import InitVar, dataclass, field from pathlib import Path +import warnings from typing import Any, Iterator, Callable from cosmos.constants import DbtResourceType, TestBehavior, ExecutionMode, LoadMode, TestIndirectSelection @@ -43,6 +44,7 @@ class RenderConfig: :param node_converters: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. :param dbt_executable_path: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. :param env_vars: A dictionary of environment variables for rendering. Only supported when using ``LoadMode.DBT_LS``. + .. deprecated:: 2.0.0 Use ProjectConfig env_vars instead. :param dbt_project_path Configures the DBT project location accessible on the airflow controller for DAG rendering. Mutually Exclusive with ProjectConfig.dbt_project_path. Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM``. """ @@ -54,12 +56,17 @@ class RenderConfig: dbt_deps: bool = True node_converters: dict[DbtResourceType, Callable[..., Any]] | None = None dbt_executable_path: str | Path = get_system_dbt() - env_vars: dict[str, str] = field(default_factory=dict) + env_vars: dict[str, str] | None = None dbt_project_path: InitVar[str | Path | None] = None project_path: Path | None = field(init=False) def __post_init__(self, dbt_project_path: str | Path | None) -> None: + if self.env_vars: + warnings.warn( + "RenderConfig.env_vars is deprecated and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", + DeprecationWarning, + ) self.project_path = Path(dbt_project_path) if dbt_project_path else None def validate_dbt_command(self, fallback_cmd: str | Path = "") -> None: @@ -96,6 +103,11 @@ class ProjectConfig: :param manifest_path: The absolute path to the dbt manifest file. Defaults to None :param project_name: Allows the user to define the project name. Required if dbt_project_path is not defined. Defaults to the folder name of dbt_project_path. + :param env_vars: Dictionary of environment variables that are used for both rendering and execution. Rendering with + env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. + :param dbt_vars: Dictionary of dbt variables for the project. This argument overrides variables defined in your dbt_project.yml + file. The dictionary is dumped to a yaml string and passed to dbt commands as the --vars argument. Variables are only + supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` load mode. """ dbt_project_path: Path | None = None @@ -113,6 +125,8 @@ def __init__( snapshots_relative_path: str | Path = "snapshots", manifest_path: str | Path | None = None, project_name: str | None = None, + env_vars: dict[str, str] | None = None, + dbt_vars: dict[str, str] | None = None, ): # Since we allow dbt_project_path to be defined in ExecutionConfig and RenderConfig # dbt_project_path may not always be defined here. @@ -136,6 +150,9 @@ def __init__( if manifest_path: self.manifest_path = Path(manifest_path) + self.env_vars = env_vars + self.dbt_vars = dbt_vars + def validate_project(self) -> None: """ Validates necessary context is present for a project. diff --git a/cosmos/converter.py b/cosmos/converter.py index 2142cc6e4..1c9620b95 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -6,6 +6,7 @@ import copy import inspect from typing import Any, Callable +from warnings import warn from airflow.models.dag import DAG from airflow.utils.task_group import TaskGroup @@ -83,6 +84,31 @@ def validate_arguments( profile_config.validate_profiles_yml() +def check_variable_args(project_config: ProjectConfig, operator_args: dict[str, Any]) -> None: + """ + Cosmos 2.0 will remove the ability to pass in operator_args with 'env' and 'vars' in place of ProjectConfig.env_vars and + ProjectConfig.dbt_vars. + """ + if "env" in operator_args: + warn( + "operator_args with 'env' is deprecated and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", + DeprecationWarning, + ) + if project_config.env_vars: + raise CosmosValueError( + "ProjectConfig.env_vars and operator_args with 'env' are mutually exclusive and only one can be used." + ) + if "vars" in operator_args: + warn( + "operator_args with 'vars' is deprecated and will be removed in Cosmos 2.0. Use ProjectConfig.vars instead.", + DeprecationWarning, + ) + if project_config.dbt_vars: + raise CosmosValueError( + "ProjectConfig.dbt_vars and operator_args with 'vars' are mutually exclusive and only one can be used." + ) + + class DbtToAirflowConverter: """ Logic common to build an Airflow DbtDag and DbtTaskGroup from a DBT project. @@ -152,6 +178,10 @@ def __init__( if not operator_args: operator_args = {} + check_variable_args(project_config, operator_args) + env_vars = project_config.env_vars or operator_args.pop("env", None) + dbt_vars = project_config.dbt_vars or operator_args.pop("vars", None) + # Previously, we were creating a cosmos.dbt.project.DbtProject # DbtProject has now been replaced with ProjectConfig directly # since the interface of the two classes were effectively the same @@ -167,7 +197,7 @@ def __init__( render_config=render_config, execution_config=execution_config, profile_config=profile_config, - operator_args=operator_args, + dbt_vars=dbt_vars, ) dbt_graph.load(method=render_config.load_method, execution_mode=execution_config.execution_mode) @@ -176,6 +206,8 @@ def __init__( "project_dir": execution_config.project_path, "profile_config": profile_config, "emit_datasets": render_config.emit_datasets, + "env": env_vars, + "vars": dbt_vars, } if execution_config.dbt_executable_path: task_args["dbt_executable_path"] = execution_config.dbt_executable_path diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index a890c137c..8f45d9e3d 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -4,6 +4,7 @@ import json import os import tempfile +import yaml from dataclasses import dataclass, field from pathlib import Path from subprocess import PIPE, Popen @@ -134,13 +135,19 @@ def __init__( render_config: RenderConfig = RenderConfig(), execution_config: ExecutionConfig = ExecutionConfig(), profile_config: ProfileConfig | None = None, - operator_args: dict[str, Any] | None = None, + # dbt_vars only supported for LegacyDbtProject + dbt_vars: dict[str, str] | None = None, ): + if project.env_vars and render_config.env_vars: + raise CosmosLoadDbtException( + "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided. RenderConfig.env_vars is deprecated, " + "please use ProjectConfig.env_vars instead." + ) self.project = project self.render_config = render_config self.profile_config = profile_config self.execution_config = execution_config - self.operator_args = operator_args or {} + self.dbt_vars = dbt_vars or {} def load( self, @@ -190,6 +197,9 @@ def run_dbt_ls( if self.render_config.select: ls_command.extend(["--select", *self.render_config.select]) + if self.project.dbt_vars: + ls_command.extend(["--vars", yaml.dump(self.project.dbt_vars)]) + ls_command.extend(self.local_flags) stdout = run_command(ls_command, tmp_dir, env_vars) @@ -235,7 +245,7 @@ def load_via_dbt_ls(self) -> None: create_symlinks(self.render_config.project_path, tmpdir_path) with self.profile_config.ensure_profile(use_mock_values=True) as profile_values, environ( - self.render_config.env_vars + self.project.env_vars or self.render_config.env_vars or {} ): (profile_path, env_vars) = profile_values env = os.environ.copy() @@ -296,7 +306,7 @@ def load_via_custom_parser(self) -> None: dbt_root_path=self.render_config.project_path.parent.as_posix(), dbt_models_dir=self.project.models_path.stem if self.project.models_path else "models", dbt_seeds_dir=self.project.seeds_path.stem if self.project.seeds_path else "seeds", - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) nodes = {} models = itertools.chain( diff --git a/cosmos/dbt/parser/project.py b/cosmos/dbt/parser/project.py index 278b1a0f7..7ca14c49c 100644 --- a/cosmos/dbt/parser/project.py +++ b/cosmos/dbt/parser/project.py @@ -130,7 +130,7 @@ class DbtModel: name: str type: DbtModelType path: Path - operator_args: Dict[str, Any] = field(default_factory=dict) + dbt_vars: Dict[str, str] = field(default_factory=dict) config: DbtModelConfig = field(default_factory=DbtModelConfig) def __post_init__(self) -> None: @@ -141,7 +141,6 @@ def __post_init__(self) -> None: return config = DbtModelConfig() - self.var_args: Dict[str, Any] = self.operator_args.get("vars", {}) code = self.path.read_text() if self.type == DbtModelType.DBT_SNAPSHOT: @@ -203,7 +202,7 @@ def _parse_jinja_ref_node(self, base_node: jinja2.nodes.Call) -> str | None: and isinstance(node.args[0], jinja2.nodes.Const) and node.node.name == "var" ): - value += self.var_args[node.args[0].value] + value += self.dbt_vars[node.args[0].value] # type: ignore elif isinstance(first_arg, jinja2.nodes.Const): # and add it to the config value = first_arg.value @@ -272,7 +271,7 @@ class LegacyDbtProject: snapshots_dir: Path = field(init=False) seeds_dir: Path = field(init=False) - operator_args: Dict[str, Any] = field(default_factory=dict) + dbt_vars: Dict[str, str] = field(default_factory=dict) def __post_init__(self) -> None: """ @@ -325,7 +324,7 @@ def _handle_csv_file(self, path: Path) -> None: name=model_name, type=DbtModelType.DBT_SEED, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) # add the model to the project self.seeds[model_name] = model @@ -343,7 +342,7 @@ def _handle_sql_file(self, path: Path) -> None: name=model_name, type=DbtModelType.DBT_MODEL, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) # add the model to the project self.models[model.name] = model @@ -353,7 +352,7 @@ def _handle_sql_file(self, path: Path) -> None: name=model_name, type=DbtModelType.DBT_SNAPSHOT, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) # add the snapshot to the project self.snapshots[model.name] = model @@ -414,7 +413,7 @@ def _extract_model_tests( name=f"{test}_{column['name']}_{model_name}", type=DbtModelType.DBT_TEST, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, config=DbtModelConfig(upstream_models=set({model_name})), ) tests[test_model.name] = test_model diff --git a/dev/dags/dbt/simple/models/top_animations.sql b/dev/dags/dbt/simple/models/top_animations.sql index 2b365b09c..cfae1c595 100644 --- a/dev/dags/dbt/simple/models/top_animations.sql +++ b/dev/dags/dbt/simple/models/top_animations.sql @@ -1,4 +1,8 @@ -{{ config(materialized='table') }} +{{ config( + materialized='table', + alias=var('animation_alias', 'top_animations') + ) +}} SELECT Title, Rating FROM {{ ref('movies_ratings_simplified') }} diff --git a/dev/dags/example_cosmos_sources.py b/dev/dags/example_cosmos_sources.py index 157b3adb3..0553b2f10 100644 --- a/dev/dags/example_cosmos_sources.py +++ b/dev/dags/example_cosmos_sources.py @@ -62,19 +62,24 @@ def convert_exposure(dag: DAG, task_group: TaskGroup, node: DbtNode, **kwargs): node_converters={ DbtResourceType("source"): convert_source, # known dbt node type to Cosmos (part of DbtResourceType) DbtResourceType("exposure"): convert_exposure, # dbt node type new to Cosmos (will be added to DbtResourceType) - }, + } +) + +# `ProjectConfig` can pass dbt variables and environment variables to dbt commands. Below is an example of +# passing a required env var for the profiles.yml file and a dbt variable that is used for rendering and +# executing dbt models. +project_config = ProjectConfig( + DBT_ROOT_PATH / "simple", env_vars={"DBT_SQLITE_PATH": DBT_SQLITE_PATH}, + dbt_vars={"animation_alias": "top_5_animated_movies"}, ) example_cosmos_sources = DbtDag( # dbt/cosmos-specific parameters - project_config=ProjectConfig( - DBT_ROOT_PATH / "simple", - ), + project_config=project_config, profile_config=profile_config, render_config=render_config, - operator_args={"env": {"DBT_SQLITE_PATH": DBT_SQLITE_PATH}}, # normal dag parameters schedule_interval="@daily", start_date=datetime(2023, 1, 1), diff --git a/docs/configuration/operator-args.rst b/docs/configuration/operator-args.rst index 9d533bf13..3283200b0 100644 --- a/docs/configuration/operator-args.rst +++ b/docs/configuration/operator-args.rst @@ -47,12 +47,10 @@ dbt-related - ``dbt_cmd_flags``: List of command flags to pass to ``dbt`` command, added after dbt subcommand - ``dbt_cmd_global_flags``: List of ``dbt`` `global flags `_ to be passed to the ``dbt`` command, before the subcommand - ``dbt_executable_path``: Path to dbt executable. -- ``env``: Declare, using a Python dictionary, values to be set as environment variables when running ``dbt`` commands. - ``fail_fast``: ``dbt`` exits immediately if ``dbt`` fails to process a resource. - ``models``: Specifies which nodes to include. - ``no_version_check``: If set, skip ensuring ``dbt``'s version matches the one specified in the ``dbt_project.yml``. - ``quiet``: run ``dbt`` in silent mode, only displaying its error logs. -- ``vars``: Supply variables to the project. This argument overrides variables defined in the ``dbt_project.yml``. - ``warn_error``: convert ``dbt`` warnings into errors. Airflow-related @@ -74,14 +72,9 @@ Sample usage "dbt_cmd_flags": ["--models", "stg_customers"], "dbt_cmd_global_flags": ["--cache-selected-only"], "dbt_executable_path": Path("/home/user/dbt"), - "env": {"MY_ENVVAR": "some-value"}, "fail_fast": True, "no_version_check": True, "quiet": True, - "vars": { - "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", - "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", - }, "warn_error": True, "cancel_query_on_kill": False, "output_enconding": "utf-8", diff --git a/docs/configuration/project-config.rst b/docs/configuration/project-config.rst index c1d952f6e..160eec492 100644 --- a/docs/configuration/project-config.rst +++ b/docs/configuration/project-config.rst @@ -1,8 +1,8 @@ Project Config ================ -The ``cosmos.config.ProjectConfig`` allows you to specify information about where your dbt project is located. It -takes the following arguments: +The ``cosmos.config.ProjectConfig`` allows you to specify information about where your dbt project is located and project +variables that should be used for rendering and execution. It takes the following arguments: - ``dbt_project_path``: The full path to your dbt project. This directory should have a ``dbt_project.yml`` file - ``models_relative_path``: The path to your models directory, relative to the ``dbt_project_path``. This defaults to @@ -16,7 +16,13 @@ takes the following arguments: - ``project_name`` : The name of the project. If ``dbt_project_path`` is provided, the ``project_name`` defaults to the folder name containing ``dbt_project.yml``. If ``dbt_project_path`` is not provided, and ``manifest_path`` is provided, ``project_name`` is required as the name can not be inferred from ``dbt_project_path`` - +- ``dbt_vars``: A dictionary of dbt variables for the project rendering and execution. This argument overrides variables + defined in the dbt_project.yml file. The dictionary of variables is dumped to a yaml string and passed to dbt commands + as the --vars argument. Variables are only supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` load mode. + Variables using `Airflow templating `_ + will only be rendered at execution time, not at render time. +- ``env_vars``: A dictionary of environment variables that are used for both rendering and execution. Rendering with + env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. Project Config Example ---------------------- @@ -31,4 +37,10 @@ Project Config Example seeds_relative_path="data", snapshots_relative_path="snapshots", manifest_path="/path/to/manifests", + env_vars={"MY_ENV_VAR": "my_env_value"}, + dbt_vars={ + "my_dbt_var": "my_value", + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, ) diff --git a/docs/configuration/render-config.rst b/docs/configuration/render-config.rst index 5e1c23824..de0a08cdb 100644 --- a/docs/configuration/render-config.rst +++ b/docs/configuration/render-config.rst @@ -14,7 +14,6 @@ The ``RenderConfig`` class takes the following arguments: - ``dbt_deps``: A Boolean to run dbt deps when using dbt ls for dag parsing. Default True - ``node_converters``: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. Find more information below. - ``dbt_executable_path``: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. -- ``env_vars``: A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. - ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` Customizing how nodes are rendered (experimental) diff --git a/tests/dbt/parser/test_project.py b/tests/dbt/parser/test_project.py index 4f13a3eb3..31fe7e18d 100644 --- a/tests/dbt/parser/test_project.py +++ b/tests/dbt/parser/test_project.py @@ -219,6 +219,6 @@ def test_dbtmodelconfig_with_vars(tmp_path): name="some_name", type=DbtModelType.DBT_MODEL, path=path_with_sources, - operator_args={"vars": {"country_code": "us"}}, + dbt_vars={"country_code": "us"}, ) assert "stg_customers_us" in dbt_model.config.upstream_models diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index a424976a1..2f490c473 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2,6 +2,7 @@ import tempfile from pathlib import Path from unittest.mock import patch +import yaml import pytest @@ -722,3 +723,88 @@ def test_parse_dbt_ls_output(): nodes = parse_dbt_ls_output(Path("fake-project"), fake_ls_stdout) assert expected_nodes == nodes + + +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_load_via_dbt_ls_project_config_env_vars(mock_validate, mock_update_nodes, mock_popen, tmp_dbt_project_dir): + """Tests that the dbt ls command in the subprocess has the project config env vars set.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + env_vars = {"MY_ENV_VAR": "my_value"} + project_config = ProjectConfig(env_vars=env_vars) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + profile_config = profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + dbt_graph.load_via_dbt_ls() + + assert "MY_ENV_VAR" in mock_popen.call_args.kwargs["env"] + assert mock_popen.call_args.kwargs["env"]["MY_ENV_VAR"] == "my_value" + + +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_load_via_dbt_ls_project_config_dbt_vars(mock_validate, mock_update_nodes, mock_popen, tmp_dbt_project_dir): + """Tests that the dbt ls command in the subprocess has "--vars" with the project config dbt_vars.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + dbt_vars = {"my_var1": "my_value1", "my_var2": "my_value2"} + project_config = ProjectConfig(dbt_vars=dbt_vars) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + profile_config = profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + dbt_graph.load_via_dbt_ls() + ls_command = mock_popen.call_args.args[0] + assert "--vars" in ls_command + assert ls_command[ls_command.index("--vars") + 1] == yaml.dump(dbt_vars) + + +@pytest.mark.sqlite +@pytest.mark.integration +def test_load_via_dbt_ls_with_project_config_vars(): + """ + Integration that tests that the dbt ls command is successful and that the node affected by the dbt_vars is + rendered correctly. + """ + project_name = "simple" + dbt_graph = DbtGraph( + project=ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, + env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")}, + dbt_vars={"animation_alias": "top_5_animated_movies"}, + ), + render_config=RenderConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, + dbt_deps=False, + ), + execution_config=ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name), + profile_config=ProfileConfig( + profile_name="simple", + target_name="dev", + profiles_yml_filepath=(DBT_PROJECTS_ROOT_DIR / project_name / "profiles.yml"), + ), + ) + dbt_graph.load_via_dbt_ls() + assert dbt_graph.nodes["model.simple.top_animations"].config["alias"] == "top_5_animated_movies" diff --git a/tests/test_config.py b/tests/test_config.py index 578a68f76..706c972c6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -174,3 +174,9 @@ def test_render_config_uses_default_if_exists(mock_which): render_config = RenderConfig(dbt_executable_path="user-dbt") render_config.validate_dbt_command("fallback-dbt-path") assert render_config.dbt_executable_path == "user-dbt" + + +def test_render_config_env_vars_deprecated(): + """RenderConfig.env_vars is deprecated, should warn user.""" + with pytest.deprecated_call(): + RenderConfig(env_vars={"VAR": "value"}) diff --git a/tests/test_converter.py b/tests/test_converter.py index 3bb5af163..11f669345 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -1,6 +1,6 @@ from datetime import datetime from pathlib import Path -from unittest.mock import patch +from unittest.mock import patch, MagicMock from cosmos.profiles.postgres import PostgresUserPasswordProfileMapping import pytest @@ -302,3 +302,62 @@ def test_converter_fails_no_manifest_no_render_config(mock_load_dbt_graph, execu err_info.value.args[0] == "RenderConfig.dbt_project_path is required for rendering an airflow DAG from a DBT Graph if no manifest is provided." ) + + +@pytest.mark.parametrize("operator_args", [{"env": {"key": "value"}}, {"vars": {"key": "value"}}]) +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.DbtGraph") +@patch("cosmos.converter.build_airflow_graph") +def test_converter_operator_args_deprecated( + mock_validate_project, mock_dbt_graph, mock_build_airflow_graph, operator_args +): + """Deprecating warnings should be raised when using operator_args with "vars" or "env".""" + project_config = ProjectConfig(project_name="fake-project", dbt_project_path="/some/project/path") + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + + with pytest.deprecated_call(): + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args=operator_args, + ) + + +@pytest.mark.parametrize("project_config_arg, operator_arg", [("dbt_vars", "vars"), ("env_vars", "env")]) +@patch("cosmos.config.ProjectConfig.validate_project") +def test_converter_fails_project_config_and_operator_args_conflict( + mock_validate_project, project_config_arg, operator_arg +): + """ + The converter should fail if a user specifies both a ProjectConfig and operator_args with dbt_vars/vars or env_vars/env + that overlap. + """ + project_config = ProjectConfig( + project_name="fake-project", + dbt_project_path="/some/project/path", + **{project_config_arg: {"key": "value"}}, # type: ignore + ) + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + operator_args = {operator_arg: {"key": "value"}} + + expected_error_msg = f"ProjectConfig.{project_config_arg} and operator_args with '{operator_arg}' are mutually exclusive and only one can be used." + with pytest.raises(CosmosValueError, match=expected_error_msg): + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args=operator_args, + ) From dbaa5e777ad7c5f46e2902ef3a1686bb358337f9 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Fri, 1 Dec 2023 12:29:38 -0800 Subject: [PATCH 02/10] update RenderConfig docstring --- cosmos/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index 05549afe7..1b62fe821 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -43,8 +43,7 @@ class RenderConfig: :param dbt_deps: Configure to run dbt deps when using dbt ls for dag parsing :param node_converters: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. :param dbt_executable_path: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. - :param env_vars: A dictionary of environment variables for rendering. Only supported when using ``LoadMode.DBT_LS``. - .. deprecated:: 2.0.0 Use ProjectConfig env_vars instead. + :param env_vars: (Deprecated use ProjectConfig.env_vars) A dictionary of environment variables for rendering. Only supported when using ``LoadMode.DBT_LS``. :param dbt_project_path Configures the DBT project location accessible on the airflow controller for DAG rendering. Mutually Exclusive with ProjectConfig.dbt_project_path. Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM``. """ From 1be61300d222b203743310ed3a41070300f8a6a5 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Tue, 5 Dec 2023 11:24:23 -0800 Subject: [PATCH 03/10] refactor tests to use validate_initial_user_config --- tests/test_converter.py | 96 +++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 61 deletions(-) diff --git a/tests/test_converter.py b/tests/test_converter.py index 11ab27c7c..5a2646218 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -44,7 +44,7 @@ def test_validate_initial_user_config_no_profile(execution_mode): profile_config = None project_config = ProjectConfig() with pytest.raises(CosmosValueError) as err_info: - validate_initial_user_config(execution_config, profile_config, project_config, None) + validate_initial_user_config(execution_config, profile_config, project_config, None, {}) err_msg = f"The profile_config is mandatory when using {execution_mode}" assert err_info.value.args[0] == err_msg @@ -57,7 +57,40 @@ def test_validate_initial_user_config_expects_profile(execution_mode): execution_config = ExecutionConfig(execution_mode=execution_mode) profile_config = None project_config = ProjectConfig() - assert validate_initial_user_config(execution_config, profile_config, project_config, None) is None + assert validate_initial_user_config(execution_config, profile_config, project_config, None, {}) is None + + +@pytest.mark.parametrize("operator_args", [{"env": {"key": "value"}}, {"vars": {"key": "value"}}]) +def test_validate_user_config_operator_args_deprecated(operator_args): + """Deprecating warnings should be raised when using operator_args with "vars" or "env".""" + project_config = ProjectConfig() + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + + with pytest.deprecated_call(): + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) + + +@pytest.mark.parametrize("project_config_arg, operator_arg", [("dbt_vars", "vars"), ("env_vars", "env")]) +def test_validate_user_config_fails_project_config_and_operator_args_overlap(project_config_arg, operator_arg): + """ + The validation should fail if a user specifies both a ProjectConfig and operator_args with dbt_vars/vars or env_vars/env + that overlap. + """ + project_config = ProjectConfig( + project_name="fake-project", + dbt_project_path="/some/project/path", + **{project_config_arg: {"key": "value"}}, # type: ignore + ) + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + operator_args = {operator_arg: {"key": "value"}} + + expected_error_msg = f"ProjectConfig.{project_config_arg} and operator_args with '{operator_arg}' are mutually exclusive and only one can be used." + with pytest.raises(CosmosValueError, match=expected_error_msg): + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) def test_validate_arguments_schema_in_task_args(): @@ -327,62 +360,3 @@ def test_converter_fails_no_manifest_no_render_config(mock_load_dbt_graph, execu err_info.value.args[0] == "RenderConfig.dbt_project_path is required for rendering an airflow DAG from a DBT Graph if no manifest is provided." ) - - -@pytest.mark.parametrize("operator_args", [{"env": {"key": "value"}}, {"vars": {"key": "value"}}]) -@patch("cosmos.config.ProjectConfig.validate_project") -@patch("cosmos.converter.DbtGraph") -@patch("cosmos.converter.build_airflow_graph") -def test_converter_operator_args_deprecated( - mock_validate_project, mock_dbt_graph, mock_build_airflow_graph, operator_args -): - """Deprecating warnings should be raised when using operator_args with "vars" or "env".""" - project_config = ProjectConfig(project_name="fake-project", dbt_project_path="/some/project/path") - execution_config = ExecutionConfig() - render_config = RenderConfig() - profile_config = MagicMock() - - with pytest.deprecated_call(): - with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: - DbtToAirflowConverter( - dag=dag, - nodes=nodes, - project_config=project_config, - profile_config=profile_config, - execution_config=execution_config, - render_config=render_config, - operator_args=operator_args, - ) - - -@pytest.mark.parametrize("project_config_arg, operator_arg", [("dbt_vars", "vars"), ("env_vars", "env")]) -@patch("cosmos.config.ProjectConfig.validate_project") -def test_converter_fails_project_config_and_operator_args_conflict( - mock_validate_project, project_config_arg, operator_arg -): - """ - The converter should fail if a user specifies both a ProjectConfig and operator_args with dbt_vars/vars or env_vars/env - that overlap. - """ - project_config = ProjectConfig( - project_name="fake-project", - dbt_project_path="/some/project/path", - **{project_config_arg: {"key": "value"}}, # type: ignore - ) - execution_config = ExecutionConfig() - render_config = RenderConfig() - profile_config = MagicMock() - operator_args = {operator_arg: {"key": "value"}} - - expected_error_msg = f"ProjectConfig.{project_config_arg} and operator_args with '{operator_arg}' are mutually exclusive and only one can be used." - with pytest.raises(CosmosValueError, match=expected_error_msg): - with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: - DbtToAirflowConverter( - dag=dag, - nodes=nodes, - project_config=project_config, - profile_config=profile_config, - execution_config=execution_config, - render_config=render_config, - operator_args=operator_args, - ) From 67f8bdfc146072642592ba75012bf12fb49f04c3 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Tue, 5 Dec 2023 17:10:16 -0800 Subject: [PATCH 04/10] move env_var check to converter and add test coverage --- cosmos/converter.py | 10 ++++++++-- cosmos/dbt/graph.py | 5 ----- tests/test_converter.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cosmos/converter.py b/cosmos/converter.py index b54fd5d3f..78453ba15 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -97,10 +97,10 @@ def validate_arguments( def validate_initial_user_config( - execution_config: ExecutionConfig | None, + execution_config: ExecutionConfig, profile_config: ProfileConfig | None, project_config: ProjectConfig, - render_config: RenderConfig | None, + render_config: RenderConfig, operator_args: dict[str, Any], ): """ @@ -146,6 +146,12 @@ def validate_initial_user_config( raise CosmosValueError( "ProjectConfig.dbt_vars and operator_args with 'vars' are mutually exclusive and only one can be used." ) + # Cosmos 2.0 will remove the ability to pass RenderConfig.env_vars in place of ProjectConfig.env_vars, check that both are not set. + if project_config.env_vars and render_config.env_vars: + raise CosmosValueError( + "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided. RenderConfig.env_vars is deprecated, " + "please use ProjectConfig.env_vars instead." + ) def validate_adapted_user_config( diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 820577544..92ef5e66f 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -138,11 +138,6 @@ def __init__( # dbt_vars only supported for LegacyDbtProject dbt_vars: dict[str, str] | None = None, ): - if project.env_vars and render_config.env_vars: - raise CosmosLoadDbtException( - "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided. RenderConfig.env_vars is deprecated, " - "please use ProjectConfig.env_vars instead." - ) self.project = project self.render_config = render_config self.profile_config = profile_config diff --git a/tests/test_converter.py b/tests/test_converter.py index 5a2646218..a8c2eaaa8 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -93,6 +93,21 @@ def test_validate_user_config_fails_project_config_and_operator_args_overlap(pro validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) +def test_validate_user_config_fails_project_config_render_config_env_vars(): + """ + The validation should fail if a user specifies both ProjectConfig.env_vars and RenderConfig.env_vars. + """ + project_config = ProjectConfig(env_vars={"key": "value"}) + execution_config = ExecutionConfig() + render_config = RenderConfig(env_vars={"key": "value"}) + profile_config = MagicMock() + operator_args = {} + + expected_error_match = "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided.*" + with pytest.raises(CosmosValueError, match=expected_error_match): + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) + + def test_validate_arguments_schema_in_task_args(): profile_config = ProfileConfig( profile_name="test", From b8f82c5f3b6de569463fa77bef3d2dc4e511f0ac Mon Sep 17 00:00:00 2001 From: Justin Bandoro <79104794+jbandoro@users.noreply.github.com> Date: Wed, 6 Dec 2023 18:01:16 -0800 Subject: [PATCH 05/10] Apply suggestions from code review Co-authored-by: Tatiana Al-Chueyr --- cosmos/config.py | 2 +- tests/dbt/test_graph.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index 1b62fe821..e79178614 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -63,7 +63,7 @@ class RenderConfig: def __post_init__(self, dbt_project_path: str | Path | None) -> None: if self.env_vars: warnings.warn( - "RenderConfig.env_vars is deprecated and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", + "RenderConfig.env_vars is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", DeprecationWarning, ) self.project_path = Path(dbt_project_path) if dbt_project_path else None diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index db8e7e231..8f794993c 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -780,7 +780,7 @@ def test_load_via_dbt_ls_project_config_env_vars(mock_validate, mock_update_node env_vars = {"MY_ENV_VAR": "my_value"} project_config = ProjectConfig(env_vars=env_vars) render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) - profile_config = profile_config = ProfileConfig( + profile_config = ProfileConfig( profile_name="test", target_name="test", profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", @@ -808,7 +808,7 @@ def test_load_via_dbt_ls_project_config_dbt_vars(mock_validate, mock_update_node dbt_vars = {"my_var1": "my_value1", "my_var2": "my_value2"} project_config = ProjectConfig(dbt_vars=dbt_vars) render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) - profile_config = profile_config = ProfileConfig( + profile_config = ProfileConfig( profile_name="test", target_name="test", profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", From 58d10a273994676e3433660208e7f39fe0c2409f Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Wed, 6 Dec 2023 18:04:06 -0800 Subject: [PATCH 06/10] add test coverage to ensure ProjectConfig.dbt_vars is used with custom load mode --- tests/test_converter.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/test_converter.py b/tests/test_converter.py index a8c2eaaa8..d84249aae 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -7,7 +7,7 @@ from airflow.models import DAG from cosmos.converter import DbtToAirflowConverter, validate_arguments, validate_initial_user_config -from cosmos.constants import DbtResourceType, ExecutionMode +from cosmos.constants import DbtResourceType, ExecutionMode, LoadMode from cosmos.config import ProjectConfig, ProfileConfig, ExecutionConfig, RenderConfig, CosmosConfigException from cosmos.dbt.graph import DbtNode from cosmos.exceptions import CosmosValueError @@ -375,3 +375,33 @@ def test_converter_fails_no_manifest_no_render_config(mock_load_dbt_graph, execu err_info.value.args[0] == "RenderConfig.dbt_project_path is required for rendering an airflow DAG from a DBT Graph if no manifest is provided." ) + + +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.build_airflow_graph") +@patch("cosmos.dbt.graph.LegacyDbtProject") +def test_converter_project_config_dbt_vars_with_custom_load_mode( + mock_legacy_dbt_project, mock_validate_project, mock_build_airflow_graph +): + """Tests that if ProjectConfig.dbt_vars are used with RenderConfig.load_method of "custom" that the + expected dbt_vars are passed to LegacyDbtProject. + """ + project_config = ProjectConfig( + project_name="fake-project", dbt_project_path="/some/project/path", dbt_vars={"key": "value"} + ) + execution_config = ExecutionConfig() + render_config = RenderConfig(load_method=LoadMode.CUSTOM) + profile_config = MagicMock() + + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args={}, + ) + _, kwargs = mock_legacy_dbt_project.call_args + assert kwargs["dbt_vars"] == {"key": "value"} From a5370ca39aee059a0135a00f78d927614e28a918 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Wed, 6 Dec 2023 18:04:53 -0800 Subject: [PATCH 07/10] docs, docstrings, and warning update to specify version of deprecation --- cosmos/config.py | 2 +- cosmos/converter.py | 6 +++--- docs/configuration/operator-args.rst | 7 +++++++ docs/configuration/project-config.rst | 4 ++-- docs/configuration/render-config.rst | 1 + tests/test_config.py | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index e79178614..92f2b2b84 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -43,7 +43,7 @@ class RenderConfig: :param dbt_deps: Configure to run dbt deps when using dbt ls for dag parsing :param node_converters: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. :param dbt_executable_path: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. - :param env_vars: (Deprecated use ProjectConfig.env_vars) A dictionary of environment variables for rendering. Only supported when using ``LoadMode.DBT_LS``. + :param env_vars: (Deprecated since Cosmos 1.3 use ProjectConfig.env_vars) A dictionary of environment variables for rendering. Only supported when using ``LoadMode.DBT_LS``. :param dbt_project_path Configures the DBT project location accessible on the airflow controller for DAG rendering. Mutually Exclusive with ProjectConfig.dbt_project_path. Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM``. """ diff --git a/cosmos/converter.py b/cosmos/converter.py index 78453ba15..c2b31700b 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -130,7 +130,7 @@ def validate_initial_user_config( # ProjectConfig.dbt_vars. if "env" in operator_args: warn( - "operator_args with 'env' is deprecated and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", + "operator_args with 'env' is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", DeprecationWarning, ) if project_config.env_vars: @@ -139,7 +139,7 @@ def validate_initial_user_config( ) if "vars" in operator_args: warn( - "operator_args with 'vars' is deprecated and will be removed in Cosmos 2.0. Use ProjectConfig.vars instead.", + "operator_args with 'vars' is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. Use ProjectConfig.vars instead.", DeprecationWarning, ) if project_config.dbt_vars: @@ -149,7 +149,7 @@ def validate_initial_user_config( # Cosmos 2.0 will remove the ability to pass RenderConfig.env_vars in place of ProjectConfig.env_vars, check that both are not set. if project_config.env_vars and render_config.env_vars: raise CosmosValueError( - "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided. RenderConfig.env_vars is deprecated, " + "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided. RenderConfig.env_vars is deprecated since Cosmos 1.3, " "please use ProjectConfig.env_vars instead." ) diff --git a/docs/configuration/operator-args.rst b/docs/configuration/operator-args.rst index 3283200b0..5ddbe6565 100644 --- a/docs/configuration/operator-args.rst +++ b/docs/configuration/operator-args.rst @@ -47,10 +47,12 @@ dbt-related - ``dbt_cmd_flags``: List of command flags to pass to ``dbt`` command, added after dbt subcommand - ``dbt_cmd_global_flags``: List of ``dbt`` `global flags `_ to be passed to the ``dbt`` command, before the subcommand - ``dbt_executable_path``: Path to dbt executable. +- ``env``: (Deprecated since Cosmos 1.3 use ``ProjectConfig.env_vars`` instead) Declare, using a Python dictionary, values to be set as environment variables when running ``dbt`` commands. - ``fail_fast``: ``dbt`` exits immediately if ``dbt`` fails to process a resource. - ``models``: Specifies which nodes to include. - ``no_version_check``: If set, skip ensuring ``dbt``'s version matches the one specified in the ``dbt_project.yml``. - ``quiet``: run ``dbt`` in silent mode, only displaying its error logs. +- ``vars``: (Deprecated since Cosmos 1.3 use ``ProjectConfig.dbt_vars`` instead) Supply variables to the project. This argument overrides variables defined in the ``dbt_project.yml``. - ``warn_error``: convert ``dbt`` warnings into errors. Airflow-related @@ -72,9 +74,14 @@ Sample usage "dbt_cmd_flags": ["--models", "stg_customers"], "dbt_cmd_global_flags": ["--cache-selected-only"], "dbt_executable_path": Path("/home/user/dbt"), + "env": {"MY_ENVVAR": "some-value"}, "fail_fast": True, "no_version_check": True, "quiet": True, + "vars": { + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, "warn_error": True, "cancel_query_on_kill": False, "output_enconding": "utf-8", diff --git a/docs/configuration/project-config.rst b/docs/configuration/project-config.rst index 160eec492..a4dd9a935 100644 --- a/docs/configuration/project-config.rst +++ b/docs/configuration/project-config.rst @@ -18,8 +18,8 @@ variables that should be used for rendering and execution. It takes the followin ``project_name`` is required as the name can not be inferred from ``dbt_project_path`` - ``dbt_vars``: A dictionary of dbt variables for the project rendering and execution. This argument overrides variables defined in the dbt_project.yml file. The dictionary of variables is dumped to a yaml string and passed to dbt commands - as the --vars argument. Variables are only supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` load mode. - Variables using `Airflow templating `_ + as the --vars argument. Variables are only supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` and + ``RenderConfig.LoadMode.CUSTOM`` load mode. Variables using `Airflow templating `_ will only be rendered at execution time, not at render time. - ``env_vars``: A dictionary of environment variables that are used for both rendering and execution. Rendering with env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. diff --git a/docs/configuration/render-config.rst b/docs/configuration/render-config.rst index de0a08cdb..c723c7084 100644 --- a/docs/configuration/render-config.rst +++ b/docs/configuration/render-config.rst @@ -14,6 +14,7 @@ The ``RenderConfig`` class takes the following arguments: - ``dbt_deps``: A Boolean to run dbt deps when using dbt ls for dag parsing. Default True - ``node_converters``: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. Find more information below. - ``dbt_executable_path``: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. +- ``env_vars``: (Deprecated since Cosmos 1.3 use ``ProjectConfig.env_vars`` instead) A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. - ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` Customizing how nodes are rendered (experimental) diff --git a/tests/test_config.py b/tests/test_config.py index 706c972c6..734303a3e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -177,6 +177,6 @@ def test_render_config_uses_default_if_exists(mock_which): def test_render_config_env_vars_deprecated(): - """RenderConfig.env_vars is deprecated, should warn user.""" + """RenderConfig.env_vars is deprecated since Cosmos 1.3, should warn user.""" with pytest.deprecated_call(): RenderConfig(env_vars={"VAR": "value"}) From 00af507b7c08bb9643ac2958e6d9b164e84fb282 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Wed, 6 Dec 2023 18:28:11 -0800 Subject: [PATCH 08/10] missed ProjectConfig docstring update --- cosmos/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos/config.py b/cosmos/config.py index 92f2b2b84..c5e7a69a3 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -106,7 +106,7 @@ class ProjectConfig: env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. :param dbt_vars: Dictionary of dbt variables for the project. This argument overrides variables defined in your dbt_project.yml file. The dictionary is dumped to a yaml string and passed to dbt commands as the --vars argument. Variables are only - supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` load mode. + supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` and ``RenderConfig.LoadMode.CUSTOM`` load mode. """ dbt_project_path: Path | None = None From c0e14e097fb4ff74ef1775a141e86733afaac7bf Mon Sep 17 00:00:00 2001 From: Justin Bandoro <79104794+jbandoro@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:14:48 -0800 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Tatiana Al-Chueyr --- docs/configuration/project-config.rst | 4 ++-- docs/configuration/render-config.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/project-config.rst b/docs/configuration/project-config.rst index a4dd9a935..c062a1de5 100644 --- a/docs/configuration/project-config.rst +++ b/docs/configuration/project-config.rst @@ -16,12 +16,12 @@ variables that should be used for rendering and execution. It takes the followin - ``project_name`` : The name of the project. If ``dbt_project_path`` is provided, the ``project_name`` defaults to the folder name containing ``dbt_project.yml``. If ``dbt_project_path`` is not provided, and ``manifest_path`` is provided, ``project_name`` is required as the name can not be inferred from ``dbt_project_path`` -- ``dbt_vars``: A dictionary of dbt variables for the project rendering and execution. This argument overrides variables +- ``dbt_vars``: (new in v1.3) A dictionary of dbt variables for the project rendering and execution. This argument overrides variables defined in the dbt_project.yml file. The dictionary of variables is dumped to a yaml string and passed to dbt commands as the --vars argument. Variables are only supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` and ``RenderConfig.LoadMode.CUSTOM`` load mode. Variables using `Airflow templating `_ will only be rendered at execution time, not at render time. -- ``env_vars``: A dictionary of environment variables that are used for both rendering and execution. Rendering with +- ``env_vars``: (new in v1.3) A dictionary of environment variables used for rendering and execution. Rendering with env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. Project Config Example diff --git a/docs/configuration/render-config.rst b/docs/configuration/render-config.rst index c723c7084..1028ecf62 100644 --- a/docs/configuration/render-config.rst +++ b/docs/configuration/render-config.rst @@ -14,7 +14,7 @@ The ``RenderConfig`` class takes the following arguments: - ``dbt_deps``: A Boolean to run dbt deps when using dbt ls for dag parsing. Default True - ``node_converters``: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. Find more information below. - ``dbt_executable_path``: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. -- ``env_vars``: (Deprecated since Cosmos 1.3 use ``ProjectConfig.env_vars`` instead) A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. +- ``env_vars``: (available in v1.2.5, use``ProjectConfig.env_vars`` for v1.3.0 onwards) A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. - ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` Customizing how nodes are rendered (experimental) From 3d2918bf72c99d90c15efc6725c29dc17312c6b2 Mon Sep 17 00:00:00 2001 From: Justin Bandoro Date: Thu, 7 Dec 2023 09:27:15 -0800 Subject: [PATCH 10/10] update pre-commit mypy to use pyproject.toml config --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3028c3ddf..b0c1aac6a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,6 +76,7 @@ repos: hooks: - id: mypy name: mypy-python + args: [--config-file, "./pyproject.toml"] additional_dependencies: [ types-PyYAML, @@ -84,7 +85,6 @@ repos: types-requests, types-python-dateutil, apache-airflow, - jinja2>=3.0.0, ] files: ^cosmos