Grant Tests Implementation (#62)

### Summary

This PR makes the necessary changes to support grants in our adapter.
All tests provided by dbt-core pass (with modifications). For grants to
work, changes need to be made to the relevant "grant" macros. Also made
a few miscellaneous changes including updating smoke_tests to not wipe
out current profiles.yml file.

### Description

#### Macros
- **support_multiple_grantees_per_dcl_statement**: Dremio only supports
one grantee per statement
- **get_show_grant_sql**: Gets data from privileges table
- **get_grant_sql** : Contains sql for granting a privilege
- **get_revoke_sql**: Contains sql for revoking a privilege
- **call_dcl_statements**: Dremio only supports calling one statement at
a time

These macros are all called by the default apply_grants macro provided
by dbt-core. For this reason, we need to make sure apply_grants is
called in each of the materializations (table, view, seed, incremental).

#### Tests
- **test_model_grants**: Tests grants for views and tables
- **test_incremental_grants**: Tests grants for incremental
materializations
- **test_seed_grants**: Tests grants for seeds
- **test_snapshot_grants**: Tests grants for snapshots
- **test_invalid_grants**: Tests invalid grants by providing a invalid
privilege and invalid user

#### Miscellaneous
- Added logic to keep current profiles.yml file in smoke_tests
- Removed the schema.sql macro, as those methods are overridden in
impl.py
- Added logic to drop a table when drop_schema is called (only spaces
allow the entire schema to be deleted)
- Added copyright headers to basic func tests

### Related Issue

#42

### Additional Reviewers

@ArgusLi @jlarue26

.github/scripts/smoke_test.sh

@@ -144,6 +144,12 @@ EOF
 test_type=$1
 test_ssl="${6:-false}"
 profiles_path=~/.dbt/profiles.yml
+temp_path=~/.dbt/temp_profiles.yml
+
+if [ -f $profiles_path ]; then
+    mv $profiles_path $temp_path
+fi
+
 
 if [ $test_type == softwareUP ]; then 
     test_user=$2
@@ -173,3 +179,7 @@ elif [ $test_type == cloud ]; then
     test_cloud
 fi
 
+if [ -f $temp_path ]; then
+  mv $temp_path $profiles_path
+fi
+

dbt/adapters/dremio/impl.py

@@ -16,12 +16,12 @@
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.dremio import DremioConnectionManager
 from dbt.adapters.dremio.relation import DremioRelation
-
+from typing import Dict
 
 from typing import List
 from typing import Optional
 from dbt.adapters.base.relation import BaseRelation
-
+from dbt.adapters.sql.impl import DROP_RELATION_MACRO_NAME
 from dbt.events import AdapterLogger
 
 logger = AdapterLogger("dremio")
@@ -66,9 +66,13 @@ def create_schema(self, relation: DremioRelation) -> None:
         self.connections.create_catalog(database, schema)
 
     def drop_schema(self, relation: DremioRelation) -> None:
-        database = relation.database
-        schema = relation.schema
-        self.connections.drop_catalog(database, schema)
+        if relation.type == "table":
+            self.execute_macro(DROP_RELATION_MACRO_NAME, kwargs={"relation": relation})
+
+        else:
+            database = relation.database
+            schema = relation.schema
+            self.connections.drop_catalog(database, schema)
 
     def timestamp_add_sql(
         self, add_to: str, number: int = 1, interval: str = "hour"
@@ -104,6 +108,30 @@ def get_rows_different_sql(
 
         return sql
 
+    def standardize_grants_dict(self, grants_table: agate.Table) -> dict:
+        """Translate the result of `show grants` (or equivalent) to match the
+        grants which a user would configure in their project.
+
+        Ideally, the SQL to show grants should also be filtering:
+        filter OUT any grants TO the current user/role (e.g. OWNERSHIP).
+        If that's not possible in SQL, it can be done in this method instead.
+
+        :param grants_table: An agate table containing the query result of
+            the SQL returned by get_show_grant_sql
+        :return: A standardized dictionary matching the `grants` config
+        :rtype: dict
+        """
+        grants_dict: Dict[str, List[str]] = {}
+        for row in grants_table:
+            # Just needed to change these two values to match Dremio cols
+            grantee = row["grantee_id"]
+            privilege = row["privilege"]
+            if privilege in grants_dict.keys():
+                grants_dict[privilege].append(grantee)
+            else:
+                grants_dict.update({privilege: [grantee]})
+        return grants_dict
+
 
 COLUMNS_EQUAL_SQL = """
 with diff_count as (

dbt/include/dremio/macros/adapters/apply_grants.sql

@@ -0,0 +1,45 @@
+/*Copyright (C) 2022 Dremio Corporation 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.*/
+
+{%- macro dremio__support_multiple_grantees_per_dcl_statement() -%}
+    {{ return(False) }}
+{%- endmacro -%}
+
+{% macro dremio__get_show_grant_sql(relation) %}
+    {%- if relation.type == 'table' -%}
+        {%- set relation_without_double_quotes = target.datalake ~ '.' ~ target.root_path ~ '.' ~ relation.identifier-%}
+    {%- else -%}
+        {%- set relation_without_double_quotes = relation.database ~ '.' ~ relation.schema ~ '.' ~ relation.identifier-%}
+    {%- endif %}
+
+    SELECT privilege, grantee_id
+        FROM sys.privileges 
+        WHERE privileges.object_id='{{ relation_without_double_quotes }}'
+{% endmacro %}
+
+{%- macro dremio__get_grant_sql(relation, privilege, grantees) -%}
+    grant {{ privilege }} on {{relation.type}} {{ relation }} to user {{ grantees | join(', ') }}
+{%- endmacro -%}
+
+{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}
+    revoke {{ privilege }} on {{ relation.type }} {{ relation }} from user {{ grantees | join(', ') }}
+{%- endmacro -%}
+
+{% macro dremio__call_dcl_statements(dcl_statement_list) %}
+    {% for dcl_statement in dcl_statement_list %}
+        {% call statement('grants') %}
+            {{ dcl_statement }};
+        {% endcall %}
+    {% endfor %}
+{% endmacro %}

dbt/include/dremio/macros/materializations/incremental/incremental.sql

@@ -37,6 +37,7 @@ limitations under the License.*/
   {%- set raw_on_schema_change = config.get('on_schema_change', validator=validation.any[basestring]) or 'ignore' -%}
   {%- set on_schema_change = incremental_validate_on_schema_change(raw_on_schema_change) -%}
   {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {% set grant_config = config.get('grants') %}
 
   {{ run_hooks(pre_hooks) }}
 
@@ -67,6 +68,8 @@ limitations under the License.*/
   {{ apply_twin_strategy(target_relation) }}
 
   {% do persist_docs(target_relation, model) %}
+
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
 
   {{ run_hooks(post_hooks) }}
 

dbt/include/dremio/macros/materializations/seed/seed.sql

@@ -21,6 +21,7 @@ limitations under the License.*/
                                                 schema=schema,
                                                 database=database,
                                                 type='table') -%}
+  {% set grant_config = config.get('grants') %}
 
   {{ run_hooks(pre_hooks) }}
 
@@ -51,6 +52,8 @@ limitations under the License.*/
 
   {% do persist_docs(target_relation, model) %}
 
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
   {{ run_hooks(post_hooks) }}
 
   {{ return({'relations': [target_relation]})}}

dbt/include/dremio/macros/materializations/table/table.sql

@@ -21,6 +21,7 @@ limitations under the License.*/
                                                 schema=schema,
                                                 database=database,
                                                 type='table') -%}
+  {% set grant_config = config.get('grants') %}
   {{ run_hooks(pre_hooks) }}
 
   -- setup: if the target relation already exists, drop it
@@ -41,6 +42,8 @@ limitations under the License.*/
 
   {% do persist_docs(target_relation, model) %}
 
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
   {{ run_hooks(post_hooks) }}
 
   {{ return({'relations': [target_relation]})}}

dbt/include/dremio/macros/materializations/view/view.sql

@@ -22,6 +22,8 @@ limitations under the License.*/
   {%- set target_relation = api.Relation.create(
       identifier=identifier, schema=schema, database=database, type='view') -%}
 
+  {% set grant_config = config.get('grants') %}
+
   {{ run_hooks(pre_hooks) }}
 
   -- If there's a table with the same name and we weren't told to full refresh,
@@ -40,6 +42,8 @@ limitations under the License.*/
 
   {{ enable_default_reflection() }}
 
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
   {{ run_hooks(post_hooks) }}
 
   {{ return({'relations': [target_relation]}) }}

tests/conftest.py

@@ -20,4 +20,6 @@ def dbt_profile_target():
         "password": os.getenv("DREMIO_PASSWORD"),
         "datalake": os.getenv("DREMIO_DATALAKE"),
         "use_ssl": False,
+        # Need to include a specific space for grants tests
+        "database": os.getenv("DREMIO_DATABASE"),
     }

tests/functional/adapter/basic/test_adapter_methods.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import pytest
 from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
 from dbt.tests.adapter.basic.test_adapter_methods import models__upstream_sql

tests/functional/adapter/basic/test_base_mat.py

@@ -1,5 +1,20 @@
 import pytest
 from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
+
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from tests.functional.adapter.utils.test_utils import (
     relation_from_name,
     check_relations_equal,
@@ -16,12 +31,12 @@
 )
 from tests.functional.adapter.utils.test_utils import DATALAKE
 
-# Unable to insert variable into docstring, so "rav-test" is hardcoded
+# Unable to insert variable into docstring, so "dbt_test_source" is hardcoded
 schema_base_yml = """
 version: 2
 sources:
   - name: raw
-    database: "rav-test" 
+    database: "dbt_test_source" 
     schema: "{{ target.schema }}"
     tables:
       - name: seed

tests/functional/adapter/basic/test_docs_generate.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import pytest
 import os
 from tests.functional.adapter.utils.test_utils import (

tests/functional/adapter/basic/test_empty.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from dbt.tests.adapter.basic.test_empty import BaseEmpty
 
 

tests/functional/adapter/basic/test_ephemeral.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import pytest
 from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
 from tests.functional.adapter.utils.test_utils import DATALAKE

tests/functional/adapter/basic/test_generic_tests.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import pytest
 from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests
 from tests.functional.adapter.utils.test_utils import DATALAKE

tests/functional/adapter/basic/test_incremental.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import pytest
 from dbt.tests.adapter.basic.test_incremental import BaseIncremental
 from tests.functional.adapter.utils.test_utils import DATALAKE

tests/functional/adapter/basic/test_singular_ephemeral.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2022 Dremio Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import pytest
 from dbt.tests.adapter.basic.test_singular_tests_ephemeral import (
     BaseSingularTestsEphemeral,