Use consistent secret scrubbing (#10006)

* replace secret placeholder with mask for consistency

* add test

* changelog

* move SECRET_PLACEHOLDER to avoid circular deps

* cleanup test

.changes/unreleased/Fixes-20240422-145811.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Use consistent secret scrubbing with the log function.
+time: 2024-04-22T14:58:11.990326-05:00
+custom:
+  Author: emmyoop
+  Issue: "9987"

core/dbt/config/renderer.py

@@ -4,9 +4,9 @@
 
 from dbt.clients.jinja import get_rendered
 from dbt_common.clients.jinja import catch_jinja
-from dbt.constants import SECRET_ENV_PREFIX, DEPENDENCIES_FILE_NAME
+from dbt.constants import SECRET_ENV_PREFIX, DEPENDENCIES_FILE_NAME, SECRET_PLACEHOLDER
 from dbt.context.target import TargetContext
-from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
+from dbt.context.secret import SecretContext
 from dbt.context.base import BaseContext
 from dbt.adapters.contracts.connection import HasCredentials
 from dbt.exceptions import DbtProjectError

core/dbt/constants.py

@@ -2,6 +2,8 @@
 SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
 DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
 
+SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
+
 MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
 MAXIMUM_SEED_SIZE_NAME = "1MB"
 

core/dbt/context/base.py

@@ -11,7 +11,7 @@
 from dbt import utils
 from dbt.clients.jinja import get_rendered
 from dbt.clients.yaml_helper import yaml, safe_load, SafeLoader, Loader, Dumper  # noqa: F401
-from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER
 from dbt.contracts.graph.nodes import Resource
 from dbt.exceptions import (
     SecretEnvVarLocationError,
@@ -561,6 +561,18 @@ def log(msg: str, info: bool = False) -> str:
               {{ log("Running some_macro: " ~ arg1 ~ ", " ~ arg2) }}
             {% endmacro %}"
         """
+        # Detect instances of the placeholder value ($$$DBT_SECRET_START...DBT_SECRET_END$$$)
+        # and replace it with the standard mask '*****'
+        if "DBT_SECRET_START" in str(msg):
+            search_group = f"({SECRET_ENV_PREFIX}(.*))"
+            pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$")
+            m = re.search(
+                pattern,
+                msg,
+            )
+            if m:
+                msg = re.sub(pattern, "*****", msg)
+
         if info:
             fire_event(JinjaLogInfo(msg=msg, node_info=get_node_info()))
         else:

core/dbt/context/secret.py

@@ -4,13 +4,10 @@
 
 from .base import BaseContext, contextmember
 
-from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
+from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER
 from dbt.exceptions import EnvVarMissingError
 
 
-SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
-
-
 class SecretContext(BaseContext):
     """This context is used in profiles.yml + packages.yml. It can render secret
     env vars that aren't usable elsewhere"""

tests/functional/dependencies/test_dependency_secrets.py

@@ -0,0 +1,31 @@
+import os
+import pytest
+
+from dbt.tests.util import run_dbt_and_capture
+from dbt_common.constants import SECRET_ENV_PREFIX
+
+
+class TestSecretInPackage:
+    @pytest.fixture(scope="class", autouse=True)
+    def setUp(self):
+        os.environ[SECRET_ENV_PREFIX + "_FOR_LOGGING"] = "super secret"
+        yield
+        del os.environ[SECRET_ENV_PREFIX + "_FOR_LOGGING"]
+
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {
+            "packages": [
+                {
+                    "package": "dbt-labs/dbt_utils{{ log(env_var('DBT_ENV_SECRET_FOR_LOGGING'), info = true) }}",
+                    "version": "1.0.0",
+                }
+            ]
+        }
+
+    def test_mask_secrets(self, project):
+        _, log_output = run_dbt_and_capture(["deps"])
+        # this will not be written to logs
+        assert not ("super secret" in log_output)
+        assert "*****" in log_output
+        assert not ("DBT_ENV_SECRET_FOR_LOGGING" in log_output)