diff --git a/CHANGELOG.md b/CHANGELOG.md index ef79df452af..6d9691b229a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Added a "docs" field to macros, with a "show" subfield to allow for hiding macros from the documentation site ([#2430](https://github.com/fishtown-analytics/dbt/issues/2430)) - Added intersection syntax for model selector ([#2167](https://github.com/fishtown-analytics/dbt/issues/2167), [#2417](https://github.com/fishtown-analytics/dbt/pull/2417)) - Extends model selection syntax with at most n-th parent/children `dbt run --models 3+m1+2` ([#2052](https://github.com/fishtown-analytics/dbt/issues/2052), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) +- Added support for renaming BigQuery relations ([#2520](https://github.com/fishtown-analytics/dbt/issues/2520), [#2521](https://github.com/fishtown-analytics/dbt/pull/2521)) ### Fixes - Fixed an error in create_adapter_plugins.py script when -dependency arg not passed ([#2507](https://github.com/fishtown-analytics/dbt/issues/2507), [#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) @@ -14,7 +15,8 @@ Contributors: - [@raalsky](https://github.com/Raalsky) ([#2417](https://github.com/fishtown-analytics/dbt/pull/2417), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431)) - [@scarrucciu](https://github.com/scarrucciu) ([#2508](https://github.com/fishtown-analytics/dbt/pull/2508)) -- [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) + - [@southpolemonkey](https://github.com/southpolemonkey)([#2511](https://github.com/fishtown-analytics/dbt/issues/2511)) + - [@azhard](https://github.com/azhard) ([#2521](https://github.com/fishtown-analytics/dbt/pull/2521) ## dbt 0.17.0 (June 08, 2020) diff --git a/Makefile b/Makefile index 9219810d4ac..5da89e89584 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test-unit: test-integration: @echo "Integration test run starting..." - @time docker-compose run test tox -e integration-postgres-py36integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36 + @time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36 test-quick: @echo "Integration test run starting..." diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index caafcb0fbfb..762e579d634 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -143,9 +143,30 @@ def truncate_relation(self, relation: BigQueryRelation) -> None: def rename_relation( self, from_relation: BigQueryRelation, to_relation: BigQueryRelation ) -> None: - raise dbt.exceptions.NotImplementedException( - '`rename_relation` is not implemented for this adapter!' - ) + + conn = self.connections.get_thread_connection() + client = conn.handle + + from_table_ref = self.connections.table_ref(from_relation.database, + from_relation.schema, + from_relation.identifier, + conn) + from_table = client.get_table(from_table_ref) + if from_table.table_type == "VIEW" or \ + from_relation.type == RelationType.View or \ + to_relation.type == RelationType.View: + raise dbt.exceptions.RuntimeException( + 'Renaming of views is not currently supported in BigQuery' + ) + + to_table_ref = self.connections.table_ref(to_relation.database, + to_relation.schema, + to_relation.identifier, + conn) + + self.cache_renamed(from_relation, to_relation) + client.copy_table(from_table_ref, to_table_ref) + client.delete_table(from_table_ref) @available def list_schemas(self, database: str) -> List[str]: diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index d6365a27653..1a06a545761 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -123,3 +123,7 @@ {% macro bigquery__alter_column_comment(relation, column_dict) -%} {% do adapter.update_column_descriptions(relation, column_dict) %} {% endmacro %} + +{% macro bigquery__rename_relation(from_relation, to_relation) -%} + {% do adapter.rename_relation(from_relation, to_relation) %} +{% endmacro %} diff --git a/test.env.sample b/test.env.sample index 9f87a3d9461..2ddf896f8f8 100644 --- a/test.env.sample +++ b/test.env.sample @@ -1,4 +1,5 @@ DBT_INVOCATION_ENV=development + SNOWFLAKE_TEST_ACCOUNT= SNOWFLAKE_TEST_USER= SNOWFLAKE_TEST_PASSWORD= @@ -11,16 +12,8 @@ SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN= SNOWFLAKE_TEST_OAUTH_CLIENT_ID= SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET= -BIGQUERY_TYPE= -BIGQUERY_PROJECT_ID= -BIGQUERY_PRIVATE_KEY_ID= -BIGQUERY_PRIVATE_KEY= -BIGQUERY_CLIENT_EMAIL= -BIGQUERY_CLIENT_ID= -BIGQUERY_AUTH_URI= -BIGQUERY_TOKEN_URI= -BIGQUERY_AUTH_PROVIDER_X509_CERT_URL= -BIGQUERY_CLIENT_X509_CERT_URL= +BIGQUERY_SERVICE_ACCOUNT_JSON= +BIGQUERY_TEST_ALT_DATABASE= REDSHIFT_TEST_HOST= REDSHIFT_TEST_USER= diff --git a/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql b/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql new file mode 100644 index 00000000000..3c030d920ec --- /dev/null +++ b/test/integration/054_adapter_methods_test/bigquery-models/renamed_model.sql @@ -0,0 +1 @@ +select * from {{ source('test_source', 'renamed_seed') }} \ No newline at end of file diff --git a/test/integration/054_adapter_methods_test/bigquery-models/sources.yml b/test/integration/054_adapter_methods_test/bigquery-models/sources.yml new file mode 100644 index 00000000000..cb74cde9c05 --- /dev/null +++ b/test/integration/054_adapter_methods_test/bigquery-models/sources.yml @@ -0,0 +1,6 @@ +version: 2 +sources: + - name: test_source + schema: "{{ target.schema }}" + tables: + - name: renamed_seed diff --git a/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql new file mode 100644 index 00000000000..253e1e0ad8a --- /dev/null +++ b/test/integration/054_adapter_methods_test/macros/rename_named_relation.sql @@ -0,0 +1,6 @@ +-- Macro to rename a relation +{% macro rename_named_relation(from_name, to_name) %} +{%- set from_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=from_name, type='table') -%} +{%- set to_relation = api.Relation.create(database=target.database, schema=target.schema, identifier=to_name, type='table') -%} +{% do adapter.rename_relation(from_relation, to_relation) %} +{% endmacro %} \ No newline at end of file diff --git a/test/integration/054_adapter_methods_test/seed_bq.sql b/test/integration/054_adapter_methods_test/seed_bq.sql new file mode 100644 index 00000000000..71a9a78c665 --- /dev/null +++ b/test/integration/054_adapter_methods_test/seed_bq.sql @@ -0,0 +1,32 @@ +create table {database}.{schema}.seed ( + id INT64, + first_name STRING, + last_name STRING, + email STRING, + gender STRING, + ip_address STRING, + updated_at TIMESTAMP +); + +-- seed inserts +insert {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values +(1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), +(2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), +(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), +(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), +(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), +(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), +(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), +(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), +(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), +(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), +(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), +(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), +(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), +(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), +(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), +(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), +(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), +(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), +(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), +(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 72d17fb0a09..c431550433b 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -1,4 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile +import yaml class TestBaseCaching(DBTIntegrationTest): @@ -40,3 +41,32 @@ def test_bigquery_adapter_methods(self): self.run_dbt(['compile']) # trigger any compile-time issues self.run_dbt() self.assertTablesEqual('model', 'expected') + + +class TestRenameRelation(DBTIntegrationTest): + @property + def schema(self): + return "rename_relation_054" + + @property + def models(self): + return 'bigquery-models' + + @property + def project_config(self): + return { + 'config-version': 2, + 'source-paths': ['models'] + } + + @use_profile('bigquery') + def test_bigquery_adapter_methods(self): + self.run_dbt(['compile']) # trigger any compile-time issues + self.run_sql_file("seed_bq.sql") + self.run_dbt(['seed']) + rename_relation_args = yaml.safe_dump({ + 'from_name': 'seed', + 'to_name': 'renamed_seed', + }) + self.run_dbt(['run-operation', 'rename_named_relation', '--args', rename_relation_args]) + self.run_dbt()