Add Grant SQL Macros

.changes/unreleased/Under the Hood-20220706-215001.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Add tests for SQL grants
+time: 2022-07-06T21:50:01.498562-04:00
+custom:
+ Author: gshank
+ Issue: "5437"
+ PR: "5447"

.github/workflows/main.yml

@@ -125,6 +125,9 @@ jobs:
  TOXENV: integration
  PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
  DBT_INVOCATION_ENV: github-actions
+ DBT_TEST_USER_1: dbt_test_user_1
+ DBT_TEST_USER_2: dbt_test_user_2
+ DBT_TEST_USER_3: dbt_test_user_3
 
  steps:
  - name: Check out the repository

.github/workflows/structured-logging-schema-check.yml

@@ -30,6 +30,11 @@ jobs:
  LOG_DIR: "/home/runner/work/dbt-core/dbt-core/logs"
  # tells integration tests to output into json format
  DBT_LOG_FORMAT: "json"
+ # Additional test users
+ DBT_TEST_USER_1: dbt_test_user_1
+ DBT_TEST_USER_2: dbt_test_user_2
+ DBT_TEST_USER_3: dbt_test_user_3
+
  steps:
  - name: checkout dev
  uses: actions/checkout@v2

core/dbt/adapters/base/impl.py

@@ -159,6 +159,7 @@ class BaseAdapter(metaclass=AdapterMeta):
  - convert_datetime_type
  - convert_date_type
  - convert_time_type
+ - standardize_grants_dict
 
  Macros:
  - get_catalog
@@ -538,6 +539,26 @@ def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[
  "`list_relations_without_caching` is not implemented for this " "adapter!"
  )
 
+ ###
+ # Methods about grants
+ ###
+ @available
+ def standardize_grants_dict(self, grants_table: agate.Table) -> dict:
+ """Translate the result of `show grants` (or equivalent) to match the
+ grants which a user would configure in their project.
+
+ If relevant -- filter down to grants made BY the current user role,
+ and filter OUT any grants TO the current user/role (e.g. OWNERSHIP).
+
+ :param grants_table: An agate table containing the query result of
+ the SQL returned by get_show_grant_sql
+ :return: A standardized dictionary matching the `grants` config
+ :rtype: dict
+ """
+ raise NotImplementedException(
+ "`standardize_grants_dict` is not implemented for this adapter!"
+ )
+
  ###
  # Provided methods about relations
  ###

core/dbt/context/base.py

@@ -657,6 +657,30 @@ def print(msg: str) -> str:
  print(msg)
  return ""
 
+ @contextmember
+ @staticmethod
+ def diff_of_two_dicts(dict_a, dict_b):
+ """
+ Given two dictionaries:
+ dict_a: {'key_x': ['value_1', 'value_2'], 'key_y': ['value_3']}
+ dict_b: {'key_x': ['value_1'], 'key_z': ['value_4']}
+ Return the same dictionary representation of dict_a MINUS dict_b
+ """
+
+ dict_diff = {}
+ dict_a = {k.lower(): v for k, v in dict_a.items()}
+ dict_b = {k.lower(): v for k, v in dict_b.items()}
+ for k in dict_a:
+ if k in dict_b:
+ a_lowered = map(lambda x: x.lower(), dict_a[k])
+ b_lowered = map(lambda x: x.lower(), dict_b[k])
+ diff = list(set(a_lowered) - set(b_lowered))
+ if diff:
+ dict_diff.update({k: diff})
+ else:
+ dict_diff.update({k: dict_a[k]})
+ return dict_diff
+
 
 def generate_base_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]:
  ctx = BaseContext(cli_vars)

core/dbt/include/global_project/macros/adapters/apply_grants.sql

@@ -0,0 +1,87 @@
+{% macro copy_grants() %}
+ {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}
+{% endmacro %}
+
+{% macro default__copy_grants() %}
+ {{ return(True) }}
+{% endmacro %}
+
+{% macro should_revoke(existing_relation, full_refresh_mode=True) %}
+
+ {% if not existing_relation %}
+ {#-- The table doesn't already exist, so no grants to copy over --#}
+ {{ return(False) }}
+ {% elif full_refresh_mode %}
+ {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}
+ {{ return(copy_grants()) }}
+ {% else %}
+ {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}
+ {{ return(True) }}
+ {% endif %}
+
+{% endmacro %}
+
+{% macro get_show_grant_sql(relation) %}
+ {{ return(adapter.dispatch("get_show_grant_sql", "dbt")(relation)) }}
+{% endmacro %}
+
+{% macro default__get_show_grant_sql(relation) %}
+ show grants on {{ relation }}
+{% endmacro %}
+
+{% macro get_grant_sql(relation, grant_config) %}
+ {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, grant_config)) }}
+{% endmacro %}
+
+{%- macro default__get_grant_sql(relation, grant_config) -%}
+ {%- for privilege in grant_config.keys() -%}
+ {%- set grantees = grant_config[privilege] -%}
+ {%- if grantees -%}
+ {%- for grantee in grantees -%}
+ grant {{ privilege }} on {{ relation }} to {{ grantee }};
+ {%- endfor -%}
+ {%- endif -%}
+ {%- endfor -%}
+{%- endmacro %}
+
+{% macro get_revoke_sql(relation, grant_config) %}
+ {{ return(adapter.dispatch("get_revoke_sql", "dbt")(relation, grant_config)) }}
+{% endmacro %}
+
+{% macro default__get_revoke_sql(relation, grant_config) %}
+ {%- for privilege in grant_config.keys() -%}
+ {%- set grantees = grant_config[privilege] -%}
+ {%- if grantees -%}
+ {%- for grantee in grantees -%}
+ revoke {{ privilege }} on {{ relation }} from {{ grantee }};
+ {% endfor -%}
+ {%- endif -%}
+ {%- endfor -%}
+{%- endmacro -%}
+
+{% macro apply_grants(relation, grant_config, should_revoke) %}
+ {{ return(adapter.dispatch("apply_grants", "dbt")(relation, grant_config, should_revoke)) }}
+{% endmacro %}
+
+{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}
+ {% if grant_config %}
+ {% if should_revoke %}
+ {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}
+ {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}
+ {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}
+ {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}
+ {% if not (needs_granting or needs_revoking) %}
+ {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}
+ {% endif %}
+ {% else %}
+ {% set needs_revoking = {} %}
+ {% set needs_granting = grant_config %}
+ {% endif %}
+ {% if needs_granting or needs_revoking %}
+ {% call statement('grants') %}
+ {{ get_revoke_sql(relation, needs_revoking) }}
+ {{ get_grant_sql(relation, needs_granting) }}
+ {% endcall %}
+ {% endif %}
+ {% endif %}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql

@@ -20,6 +20,8 @@
  -- BEGIN, in a separate transaction
  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+ -- grab current tables grants config for comparision later on
+ {% set grant_config = config.get('grants') %}
  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
@@ -59,6 +61,9 @@
  {% do to_drop.append(backup_relation) %}
  {% endif %}
 
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
  {% do persist_docs(target_relation, model) %}
 
  {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}

core/dbt/include/global_project/macros/materializations/models/table/table.sql

@@ -14,6 +14,8 @@
  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
  -- as above, the backup_relation should not already exist
  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+ -- grab current tables grants config for comparision later on
+ {% set grant_config = config.get('grants') %}
 
  -- drop the temp relations if they exist already in the database
  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
@@ -40,6 +42,9 @@
 
  {{ run_hooks(post_hooks, inside_transaction=True) }}
 
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
  {% do persist_docs(target_relation, model) %}
 
  -- `COMMIT` happens here

core/dbt/include/global_project/macros/materializations/models/view/create_or_replace_view.sql

@@ -13,12 +13,12 @@
  {%- set identifier = model['alias'] -%}
 
  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
-
  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
 
  {%- set target_relation = api.Relation.create(
  identifier=identifier, schema=schema, database=database,
  type='view') -%}
+ {% set grant_config = config.get('grants') %}
 
  {{ run_hooks(pre_hooks) }}
 
@@ -34,6 +34,9 @@
  {{ get_create_view_as_sql(target_relation, sql) }}
  {%- endcall %}
 
+ {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=True) %}
+
  {{ run_hooks(post_hooks) }}
 
  {{ return({'relations': [target_relation]}) }}

core/dbt/include/global_project/macros/materializations/models/view/view.sql

@@ -25,6 +25,8 @@
  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
  -- as above, the backup_relation should not already exist
  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+ -- grab current tables grants config for comparision later on
+ {% set grant_config = config.get('grants') %}
 
  {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -47,6 +49,9 @@
  {% endif %}
  {{ adapter.rename_relation(intermediate_relation, target_relation) }}
 
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
  {% do persist_docs(target_relation, model) %}
 
  {{ run_hooks(post_hooks, inside_transaction=True) }}

core/dbt/include/global_project/macros/materializations/seeds/seed.sql

@@ -8,7 +8,10 @@
  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
 
+ {%- set grant_config = config.get('grants') -%}
  {%- set agate_table = load_agate_table() -%}
+ -- grab current tables grants config for comparision later on
+
  {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}
 
  {{ run_hooks(pre_hooks, inside_transaction=False) }}
@@ -35,6 +38,10 @@
  {% endcall %}
 
  {% set target_relation = this.incorporate(type='table') %}
+
+ {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
  {% do persist_docs(target_relation, model) %}
 
  {% if full_refresh_mode or not exists_as_table %}

core/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql

@@ -5,6 +5,8 @@
 
  {%- set strategy_name = config.get('strategy') -%}
  {%- set unique_key = config.get('unique_key') %}
+ -- grab current tables grants config for comparision later on
+ {%- set grant_config = config.get('grants') -%}
 
  {% set target_relation_exists, target_relation = get_or_create_relation(
  database=model.database,
@@ -73,6 +75,9 @@
  {{ final_sql }}
  {% endcall %}
 
+ {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
  {% do persist_docs(target_relation, model) %}
 
  {% if not target_relation_exists %}

core/dbt/parser/manifest.py

@@ -634,7 +634,9 @@ def read_manifest_for_partial_parse(self) -> Optional[Manifest]:
  if not flags.PARTIAL_PARSE:
  fire_event(PartialParsingNotEnabled())
  return None
- path = os.path.join(self.root_project.target_path, PARTIAL_PARSE_FILE_NAME)
+ path = os.path.join(
+ self.root_project.project_root, self.root_project.target_path, PARTIAL_PARSE_FILE_NAME
+ )
 
  reparse_reason = None
 

core/dbt/tests/util.py

@@ -73,7 +73,10 @@ def run_dbt(args: List[str] = None, expect_pass=True):
  return res
 
 
-# Use this if you need to capture the command logs in a test
+# Use this if you need to capture the command logs in a test.
+# If you want the logs that are normally written to a file, you must
+# start with the "--debug" flag. The structured schema log CI test
+# will turn the logs into json, so you have to be prepared for that.
 def run_dbt_and_capture(args: List[str] = None, expect_pass=True):
  try:
  stringbuf = capture_stdout_logs()
@@ -83,6 +86,11 @@ def run_dbt_and_capture(args: List[str] = None, expect_pass=True):
  finally:
  stop_capture_stdout_logs()
 
+ # Json logs will have lots of escape characters which will
+ # make checks for strings in the logs fail, so remove those.
+ if '{"code":' in stdout:
+ stdout = stdout.replace("\\", "")
+
  return res, stdout
 
 

plugins/postgres/dbt/adapters/postgres/impl.py

@@ -10,6 +10,7 @@
 from dbt.dataclass_schema import dbtClassMixin, ValidationError
 import dbt.exceptions
 import dbt.utils
+import agate
 
 
 # note that this isn't an adapter macro, so just a single underscore
@@ -85,6 +86,18 @@ def verify_database(self, database):
  def parse_index(self, raw_index: Any) -> Optional[PostgresIndexConfig]:
  return PostgresIndexConfig.parse(raw_index)
 
+ @available
+ def standardize_grants_dict(self, grants_table: agate.Table) -> dict:
+ grants_dict = {}
+ for row in grants_table:
+ grantee = row["grantee"]
+ privilege = row["privilege_type"]
+ if privilege in grants_dict.keys():
+ grants_dict[privilege].append(grantee)
+ else:
+ grants_dict.update({privilege: [grantee]})
+ return grants_dict
+
  def _link_cached_database_relations(self, schemas: Set[str]):
  """
  :param schemas: The set of schemas that should have links added.

plugins/postgres/dbt/include/postgres/macros/adapters.sql

@@ -202,3 +202,16 @@
  comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};
  {% endfor %}
 {% endmacro %}
+
+{%- macro postgres__get_show_grant_sql(relation) -%}
+ select grantee, privilege_type
+ from information_schema.role_table_grants
+ where grantor = current_role
+ and grantee != current_role
+ and table_schema = '{{ relation.schema }}'
+ and table_name = '{{ relation.identifier }}'
+{%- endmacro -%}
+
+{% macro postgres__are_grants_copied_over_when_replaced() %}
+ {{ return(False) }}
+{% endmacro %}

test/setup_db.sh

@@ -46,6 +46,9 @@ psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
 psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
 psql -c "ALTER ROLE noaccess WITH LOGIN;"
 psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
+psql -c "CREATE ROLE dbt_test_user_1;"
+psql -c "CREATE ROLE dbt_test_user_2;"
+psql -c "CREATE ROLE dbt_test_user_3;"
 
 psql -c 'CREATE DATABASE "dbtMixedCase";'
 psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;'