Merge pull request #45 from dbt-labs/dev/0.4.0

Merge dev/0.4.0 into main

.circleci/config.yml

@@ -25,7 +25,7 @@ jobs:
             . venv/bin/activate
 
             pip install --upgrade pip setuptools
-            pip install dbt --upgrade
+            pip install --pre dbt --upgrade
 
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml

.github/pull_request_template.md

@@ -1,3 +1,8 @@
+This is a:
+- [ ] bug fix PR with no breaking changes — please ensure the base branch is `main`
+- [ ] new functionality — please ensure the base branch is the latest `dev/` branch
+- [ ] a breaking change — please ensure the base branch is the latest `dev/` branch
+
 ## Description & motivation
 <!---
 Describe your changes, and why you're making them.
@@ -7,3 +12,4 @@ Describe your changes, and why you're making them.
 - [ ] I have verified that these changes work locally
 - [ ] I have updated the README.md (if applicable)
 - [ ] I have added tests & descriptions to my models (and macros if applicable)
+- [ ] I have added an entry to CHANGELOG.md

CHANGELOG.md

@@ -0,0 +1,75 @@
+# dbt-codegen v0.4.0
+
+## Breaking changes
+- Requires `dbt>=0.20.0` and `dbt-utils>=0.7.0`
+- Depends on `dbt-labs/dbt_utils` (instead of `fishtown-analytics/dbt_utils`)
+
+## Features
+- Add optional `leading_commas` arg to `generate_base_model` (#41 @jaypeedevlin)
+- Add optional `include_descriptions` arg to `generate_source` (#40 @djbelknapdbs)
+
+## Fixes
+- In the `generate_source` macro, use `dbt_utils.get_relations_by_pattern` instead of `get_relations_by_prefix`, since the latter will be deprecated in the future (#42)
+
+## Under the hood
+- Use new adapter.dispatch syntax (#44)
+
+# dbt-codegen v0.3.2
+
+This is a quality of life release
+
+## Other
+* Fix rendering issues on hub.getdbt.com
+* Fix integration tests due to python version compatibility
+
+# dbt-codegen v0.3.1
+This is a bugfix release
+
+## Fixes
+- Use latest version of dbt-utils (0.6.2) to ensure generate_source_yaml works for non-target schemata (#34)
+
+# dbt-codegen v0.3.0
+## 🚨  Breaking change
+This release requires dbt v0.18.0, and dbt-utils v0.6.1. If you're not ready to upgrade, consider using a previous release of this package.
+
+## Quality of life
+- Use dbt v0.18.0 (#31)
+- Fix README rendering on hub  (#32 @calvingiles)
+
+# dbt-codegen v0.2.0
+## 🚨 Breaking change
+The lower bound of `dbt-utils` is now `0.4.0`.
+
+This won't affect most users, since you're likely already using version of dbt-utils higher than this to achieve 0.17.0 compatibility.
+
+## Quality of life:
+- Change dbt-utils dependencies to `[>=0.4.0, <0.6.0]` (#29)
+- Fix tests (#29)
+
+# dbt-codegen v0.1.0
+## 🚨 Breaking change!
+
+This package now requires dbt v0.17.x!
+
+## Features:
+* Add `generate_model_yaml` (#18 @jtalmi)
+
+
+## Under the hood:
+* Update to v0.17.0, including `dbt_project.yml` version 2 syntax (#23)
+* Add GitHub templates and installation instructions (#23)
+
+## Acknowledgements
+@marzaccaro made a PR for `generate_model_yaml`, and, although I had reviewed it, I let the PR go stale and somehow completely forgot about it when merging PR #18 — this is completely my bad! So equal credit to @marzaccaro and @jtalmi for their work :clap:
+
+# dbt-codegen v0.0.4
+This is a bugfix release to improve compatibility with Snowflake
+
+# dbt-codegen v0.0.3
+Bump utils version range
+
+# dbt-codegen v0.0.2
+Small quality of life improvements
+
+# dbt-codegen v0.0.1
+Initial release

README.md

@@ -9,7 +9,7 @@ Macros that generate dbt code, and log it to the command line.
 
 # Installation instructions
 New to dbt packages? Read more about them [here](https://docs.getdbt.com/docs/building-a-dbt-project/package-management/).
-1. Include this package in your `packages.yml` file — check [here](https://hub.getdbt.com/fishtown-analytics/codegen/latest/) for the latest version number.
+1. Include this package in your `packages.yml` file — check [here](https://hub.getdbt.com/dbt-labs/codegen/latest/) for the latest version number.
 2. Run `dbt deps` to install the package.
 
 # Macros
@@ -23,6 +23,8 @@ which you can then paste into a schema file.
 source data is in.
 * `generate_columns` (optional, default=False): Whether you want to add the
 column names to your source definition.
+* `include_descriptions` (optional, default=False): Whether you want to add 
+description placeholders to your source definition.
 
 ### Usage:
 1. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code
@@ -54,8 +56,11 @@ sources:
     database: raw
     tables:
       - name: customers
+        description: ""
       - name: orders
+        description: ""
       - name: payments
+        description: ""
 ```
 
 3. Paste the output in to a schema `.yml` file, and refactor as required.
@@ -67,6 +72,7 @@ model.
 ### Arguments:
 * `source_name` (required): The source you wish to generate base model SQL for.
 * `table_name` (required): The source table you wish to generate base model SQL for.
+* `leading_commas` (optional, default=False): Whether you want your commas to be leading (vs trailing).
 
 
 ### Usage:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'codegen'
-version: '1.0'
+version: '0.4.0'
 
 require-dbt-version: ">=0.18.0"
 config-version: 2

integration_tests/tests/test_generate_base_models_leading.sql

@@ -0,0 +1,29 @@
+
+{% set actual_base_model = codegen.generate_base_model(
+    source_name='codegen_integration_tests__data_source_schema',
+    table_name='codegen_integration_tests__data_source_table',
+    leading_commas=True,
+  )
+%}
+
+{% set expected_base_model %}
+with source as (
+
+    select * from {%raw%}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table') }}{%endraw%}
+
+),
+
+renamed as (
+
+    select
+        my_integer_col
+        , my_bool_col
+
+    from source
+
+)
+
+select * from renamed
+{% endset %}
+
+{{ assert_equal (actual_base_model | trim, expected_base_model | trim) }}

integration_tests/tests/test_generate_source_all_args.sql

@@ -4,7 +4,8 @@
 {% set actual_source_yaml = codegen.generate_source(
     schema_name=raw_schema,
     database_name=target.database,
-    generate_columns=True
+    generate_columns=True,
+    include_descriptions=True
 ) %}
 
 
@@ -17,7 +18,9 @@ sources:
       - name: data__a_relation
         columns:
           - name: col_a
+            description: ""
           - name: col_b
+            description: ""
 
 {% endset %}
 

macros/generate_base_model.sql

@@ -1,4 +1,4 @@
-{% macro generate_base_model(source_name, table_name) %}
+{% macro generate_base_model(source_name, table_name, leading_commas=False) %}
 
 {%- set source_relation = source(source_name, table_name) -%}
 
@@ -14,9 +14,15 @@ with source as (
 renamed as (
 
     select
+        {%- if leading_commas -%}
         {%- for column in column_names %}
-        {{ column | lower }}{{"," if not loop.last}}
+        {{", " if not loop.first}}{{ column | lower }}
         {%- endfor %}
+        {%- else -%}
+        {%- for column in column_names %}
+        {{ column | lower }}{{"," if not loop.last}}
+        {%- endfor -%}
+        {%- endif %}
 
     from source
 

macros/generate_source.sql

@@ -1,23 +1,20 @@
-{% macro get_tables_in_schema(schema_name,database_name=target.database) %}
+{% macro get_tables_in_schema(schema_name, database_name=target.database) %}
 
-    {% set tables=dbt_utils.get_relations_by_prefix(
-            schema=schema_name,
-            prefix='',
-            database=database_name
-        )
-    %}
+    {% set tables=dbt_utils.get_relations_by_pattern(
+            database=database_name,
+            schema_pattern=schema_name,
+            table_pattern='%'
+    ) %}
 
-    {% set table_list= tables | map(attribute = 'identifier') %}
+    {% set table_list= tables | map(attribute='identifier') %}
 
     {{ return(table_list | sort) }}
-    {{ log("*** table list ***", info=True) }}
-    {{ log(table_list, info=True) }}
 
 {% endmacro %}
 
 
 ---
-{% macro generate_source(schema_name, database_name=target.database, generate_columns=False) %}
+{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False) %}
 
 {% set sources_yaml=[] %}
 
@@ -50,6 +47,9 @@
 
         {% for column in columns %}
             {% do sources_yaml.append('          - name: ' ~ column.name | lower ) %}
+            {% if include_descriptions %}
+                {% do sources_yaml.append('            description: ""' ) %}
+            {% endif %}
         {% endfor %}
             {% do sources_yaml.append('') %}
 

packages.yml

@@ -1,3 +1,3 @@
 packages:
-  - package: fishtown-analytics/dbt_utils
-    version: [">=0.6.2", "<0.7.0"]
+  - package: dbt-labs/dbt_utils
+    version: [">=0.6.2", "<0.8.0"]