Store when default env vars are used in manifest (#5589)

* WIP

* handle defauly env vars

* fix typo

* add changelog

* small fixes

* add constants.py file

.changes/unreleased/Under the Hood-20220802-112936.yaml

@@ -0,0 +1,8 @@
+kind: Under the Hood
+body: Save use of default env vars to manifest to enable partial parsing in those
+ cases.
+time: 2022-08-02T11:29:36.417589-05:00
+custom:
+ Author: emmyoop
+ Issue: "5155"
+ PR: "5589"

core/dbt/config/renderer.py

@@ -3,13 +3,13 @@
 import os
 
 from dbt.clients.jinja import get_rendered, catch_jinja
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.context.target import TargetContext
 from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
 from dbt.context.base import BaseContext
 from dbt.contracts.connection import HasCredentials
 from dbt.exceptions import DbtProjectError, CompilationException, RecursionException
 from dbt.utils import deep_map_render
-from dbt.logger import SECRET_ENV_PREFIX
 
 
 Keypath = Tuple[Union[str, int], ...]

core/dbt/constants.py

@@ -0,0 +1,2 @@
+SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
+DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"

core/dbt/context/base.py

@@ -6,6 +6,7 @@
 from dbt import tracking
 from dbt.clients.jinja import get_rendered
 from dbt.clients.yaml_helper import yaml, safe_load, SafeLoader, Loader, Dumper # noqa: F401
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.contracts.graph.compiled import CompiledResource
 from dbt.exceptions import (
  CompilationException,
@@ -14,7 +15,6 @@
  raise_parsing_error,
  disallow_secret_env_var,
 )
-from dbt.logger import SECRET_ENV_PREFIX
 from dbt.events.functions import fire_event, get_invocation_id
 from dbt.events.types import MacroEventInfo, MacroEventDebug
 from dbt.version import __version__ as dbt_version
@@ -305,7 +305,12 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
  return_value = default
 
  if return_value is not None:
- self.env_vars[var] = return_value
+ # If the environment variable is set from a default, store a string indicating
+ # that so we can skip partial parsing. Otherwise the file will be scheduled for
+ # reparsing. If the default changes, the file will have been updated and therefore
+ # will be scheduled for reparsing anyways.
+ self.env_vars[var] = return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+
  return return_value
  else:
  msg = f"Env var required but not provided: '{var}'"

core/dbt/context/configured.py

@@ -1,8 +1,8 @@
 import os
 from typing import Any, Dict, Optional
 
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.contracts.connection import AdapterRequiredConfig
-from dbt.logger import SECRET_ENV_PREFIX
 from dbt.node_types import NodeType
 from dbt.utils import MultiDict
 
@@ -94,7 +94,14 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
 
  if return_value is not None:
  if self.schema_yaml_vars:
- self.schema_yaml_vars.env_vars[var] = return_value
+ # If the environment variable is set from a default, store a string indicating
+ # that so we can skip partial parsing. Otherwise the file will be scheduled for
+ # reparsing. If the default changes, the file will have been updated and therefore
+ # will be scheduled for reparsing anyways.
+ self.schema_yaml_vars.env_vars[var] = (
+ return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+ )
+
  return return_value
  else:
  msg = f"Env var required but not provided: '{var}'"

core/dbt/context/providers.py

@@ -22,7 +22,7 @@
 from .base import contextmember, contextproperty, Var
 from .configured import FQNLookup
 from .context_config import ContextConfig
-from dbt.logger import SECRET_ENV_PREFIX
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
 from .macros import MacroNamespaceBuilder, MacroNamespace
 from .manifest import ManifestContext
@@ -1211,7 +1211,14 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
  # Save the env_var value in the manifest and the var name in the source_file.
  # If this is compiling, do not save because it's irrelevant to parsing.
  if self.model and not hasattr(self.model, "compiled"):
- self.manifest.env_vars[var] = return_value
+ # If the environment variable is set from a default, store a string indicating
+ # that so we can skip partial parsing. Otherwise the file will be scheduled for
+ # reparsing. If the default changes, the file will have been updated and therefore
+ # will be scheduled for reparsing anyways.
+ self.manifest.env_vars[var] = (
+ return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+ )
+
  # hooks come from dbt_project.yml which doesn't have a real file_id
  if self.model.file_id in self.manifest.files:
  source_file = self.manifest.files[self.model.file_id]
@@ -1535,7 +1542,13 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
  if return_value is not None:
  # Save the env_var value in the manifest and the var name in the source_file
  if self.model:
- self.manifest.env_vars[var] = return_value
+ # If the environment variable is set from a default, store a string indicating
+ # that so we can skip partial parsing. Otherwise the file will be scheduled for
+ # reparsing. If the default changes, the file will have been updated and therefore
+ # will be scheduled for reparsing anyways.
+ self.manifest.env_vars[var] = (
+ return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
+ )
  # the "model" should only be test nodes, but just in case, check
  # TODO CT-211
  if self.model.resource_type == NodeType.Test and self.model.file_key_name: # type: ignore[union-attr] # noqa

core/dbt/context/secret.py

@@ -3,8 +3,8 @@
 
 from .base import BaseContext, contextmember
 
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.exceptions import raise_parsing_error
-from dbt.logger import SECRET_ENV_PREFIX
 
 
 SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
@@ -43,7 +43,11 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
  # if it's a 'secret' env var, we shouldn't even get here
  # but just to be safe — don't save secrets
  if not var.startswith(SECRET_ENV_PREFIX):
- self.env_vars[var] = return_value
+ # If the environment variable is set from a default, store a string indicating
+ # that so we can skip partial parsing. Otherwise the file will be scheduled for
+ # reparsing. If the default changes, the file will have been updated and therefore
+ # will be scheduled for reparsing anyways.
+ self.env_vars[var] = return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER
  return return_value
  else:
  msg = f"Env var required but not provided: '{var}'"

core/dbt/events/functions.py

@@ -3,9 +3,10 @@
 from dbt.events.base_types import NoStdOut, Event, NoFile, ShowException, Cache
 from dbt.events.types import EventBufferFull, T_Event, MainReportVersion, EmptyLine
 import dbt.flags as flags
+from dbt.constants import SECRET_ENV_PREFIX
 
 # TODO this will need to move eventually
-from dbt.logger import SECRET_ENV_PREFIX, make_log_dir_if_missing, GLOBAL_LOGGER
+from dbt.logger import make_log_dir_if_missing, GLOBAL_LOGGER
 from datetime import datetime
 import json
 import io

core/dbt/logger.py

@@ -13,6 +13,7 @@
 
 import colorama
 import logbook
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.dataclass_schema import dbtClassMixin
 
 # Colorama is needed for colored logs on Windows because we're using logger.info
@@ -31,8 +32,6 @@
  "{record.time:%Y-%m-%d %H:%M:%S.%f%z} " "({record.thread_name}): " "{record.message}"
 )
 
-SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
-
 
 def get_secret_env() -> List[str]:
  return [v for k, v in os.environ.items() if k.startswith(SECRET_ENV_PREFIX)]

core/dbt/parser/partial.py

@@ -21,6 +21,7 @@
  PartialParsingDeletedExposure,
  PartialParsingDeletedMetric,
 )
+from dbt.constants import DEFAULT_ENV_PLACEHOLDER
 from dbt.node_types import NodeType
 
 
@@ -961,6 +962,13 @@ def build_env_vars_to_files(self):
  prev_value = self.saved_manifest.env_vars[env_var]
  current_value = os.getenv(env_var)
  if current_value is None:
+ # This will be true when depending on the default value.
+ # We store env vars set by defaults as a static string so we can recognize they have
+ # defaults. We depend on default changes triggering reparsing by file change. If
+ # the file has not changed we can assume the default has not changed.
+ if prev_value == DEFAULT_ENV_PLACEHOLDER:
+ unchanged_vars.append(env_var)
+ continue
  # env_var no longer set, remove from manifest
  delete_vars.append(env_var)
  if prev_value == current_value:

test/integration/068_partial_parsing_tests/test_pp_vars.py

@@ -1,8 +1,8 @@
 from dbt.exceptions import CompilationException, ParsingException
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.files import ParseFileType
 from dbt.contracts.results import TestStatus
-from dbt.logger import SECRET_ENV_PREFIX
 from dbt.parser.partial import special_override_macros
 from test.integration.base import DBTIntegrationTest, use_profile, normalize, get_manifest
 import shutil

tests/functional/context_methods/test_env_vars.py

@@ -1,8 +1,8 @@
 import pytest
 import os
 
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
 from dbt.tests.util import run_dbt, get_manifest, run_dbt_and_capture
-from dbt.logger import SECRET_ENV_PREFIX
 
 
 context_sql = """
@@ -36,6 +36,8 @@
  '{{ invocation_id }}' as invocation_id,
 
  '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var,
+ '{{ env_var("DBT_TEST_IGNORE_DEFAULT", "ignored_default_val") }}' as env_var_ignore_default,
+ '{{ env_var("DBT_TEST_USE_DEFAULT", "use_my_default_val") }}' as env_var_use_default,
  'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs
  '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret
 
@@ -54,11 +56,13 @@ def setup(self):
  os.environ["DBT_TEST_PASS"] = "password"
  os.environ[SECRET_ENV_PREFIX + "SECRET"] = "secret_variable"
  os.environ["DBT_TEST_NOT_SECRET"] = "regular_variable"
+ os.environ["DBT_TEST_IGNORE_DEFAULT"] = "ignored_default"
  yield
  del os.environ["DBT_TEST_ENV_VAR"]
  del os.environ["DBT_TEST_USER"]
  del os.environ[SECRET_ENV_PREFIX + "SECRET"]
  del os.environ["DBT_TEST_NOT_SECRET"]
+ del os.environ["DBT_TEST_IGNORE_DEFAULT"]
 
  @pytest.fixture(scope="class")
  def profiles_config_update(self, unique_schema):
@@ -129,7 +133,12 @@ def test_env_vars_dev(
  ctx = self.get_ctx_vars(project)
 
  manifest = get_manifest(project.project_root)
- expected = {"DBT_TEST_ENV_VAR": "1", "DBT_TEST_NOT_SECRET": "regular_variable"}
+ expected = {
+ "DBT_TEST_ENV_VAR": "1",
+ "DBT_TEST_NOT_SECRET": "regular_variable",
+ "DBT_TEST_IGNORE_DEFAULT": "ignored_default",
+ "DBT_TEST_USE_DEFAULT": DEFAULT_ENV_PLACEHOLDER,
+ }
  assert manifest.env_vars == expected
 
  this = '"{}"."{}"."context"'.format(project.database, project.test_schema)

tests/functional/context_methods/test_secret_env_vars.py

@@ -1,10 +1,10 @@
 import pytest
 import os
 
-from dbt.tests.util import run_dbt, run_dbt_and_capture
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.exceptions import ParsingException, InternalException
-from dbt.logger import SECRET_ENV_PREFIX
 from tests.functional.context_methods.first_dependency import FirstDependencyProject
+from dbt.tests.util import run_dbt, run_dbt_and_capture
 
 
 secret_bad__context_sql = """