fewer adapters will need to re-implemnt basic_load_csv_rows (dbt-labs…

…#3623) * fewer adapters will need to re-implemnt basic_load_csv_rows * hack version * reordering per convention * make redundant basic_load_csv_rows * for next version * Update core/dbt/include/global_project/macros/materializations/seed/seed.sql Co-authored-by: Jeremy Cohen <jtcohen6@gmail.com> * Move up changelog entry Co-authored-by: Jeremy Cohen <jtcohen6@gmail.com> Co-authored-by: Jeremy Cohen <jeremy@fishtownanalytics.com>
TeddyCr · Sep 9, 2021 · 6f495a8 · 6f495a8
1 parent 946fdd8
commit 6f495a8
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 38 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,10 +10,12 @@
 
 - Use GitHub Actions for CI ([#3688](https://github.com/dbt-labs/dbt/issues/3688), [#3669](https://github.com/dbt-labs/dbt/pull/3669))
 - Better dbt hub registry packages version logging that prompts the user for upgrades to relevant packages ([#3560](https://github.com/dbt-labs/dbt/issues/3560), [#3763](https://github.com/dbt-labs/dbt/issues/3763), [#3759](https://github.com/dbt-labs/dbt/pull/3759))
+- Allow the default seed macro's SQL parameter, `%s`, to be replaced by dispatching a new macro, `get_binding_char()`. This enables adapters with parameter marker characters such as `?` to not have to override `basic_load_csv_rows`. ([#3622](https://github.com/fishtown-analytics/dbt/issues/3622), [#3623](https://github.com/fishtown-analytics/dbt/pull/3623))
 
 Contributors:
 - [@sungchun12](https://github.com/sungchun12) ([#3759](https://github.com/dbt-labs/dbt/pull/3759))
 - [@xemuliam](https://github.com/xemuliam) ([#3606](https://github.com/dbt-labs/dbt/pull/3606))
+- [@swanderz](https://github.com/swanderz) [#3623](https://github.com/fishtown-analytics/dbt/pull/3623)
 
 ## dbt 0.21.0b2 (August 19, 2021)
 

diff --git a/core/dbt/include/global_project/macros/materializations/seed/seed.sql b/core/dbt/include/global_project/macros/materializations/seed/seed.sql
@@ -3,14 +3,6 @@
  {{ adapter.dispatch('create_csv_table')(model, agate_table) }}
 {%- endmacro %}
 
-{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
- {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}
-{%- endmacro %}
-
-{% macro load_csv_rows(model, agate_table) -%}
- {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}
-{%- endmacro %}
-
 {% macro default__create_csv_table(model, agate_table) %}
  {%- set column_override = model['config'].get('column_types', {}) -%}
  {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
@@ -33,6 +25,9 @@
  {{ return(sql) }}
 {% endmacro %}
 
+{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
+ {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}
+{%- endmacro %}
 
 {% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
  {% set sql = "" %}
@@ -47,6 +42,21 @@
  {{ return(sql) }}
 {% endmacro %}
 
+{% macro get_binding_char() -%}
+ {{ adapter.dispatch('get_binding_char')() }}
+{%- endmacro %}
+
+{% macro default__get_binding_char() %}
+ {{ return('%s') }}
+{% endmacro %}
+
+{% macro get_batch_size() -%}
+ {{ adapter.dispatch('get_batch_size')() }}
+{%- endmacro %}
+
+{% macro default__get_batch_size() %}
+ {{ return(10000) }}
+{% endmacro %}
 
 {% macro get_seed_column_quoted_csv(model, column_names) %}
  {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
@@ -59,48 +69,48 @@
  {{ return(dest_cols_csv) }}
 {% endmacro %}
 
+{% macro load_csv_rows(model, agate_table) -%}
+ {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}
+{%- endmacro %}
 
-{% macro basic_load_csv_rows(model, batch_size, agate_table) %}
- {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
- {% set bindings = [] %}
+{% macro default__load_csv_rows(model, agate_table) %}
 
-  {% set statements = [] %}
+ {% set batch_size = get_batch_size() %}
 
-  {% for chunk in agate_table.rows | batch(batch_size) %}
-  {% set bindings = [] %}
+ {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+ {% set bindings = [] %}
 
- {% for row in chunk %}
- {% do bindings.extend(row) %}
- {% endfor %}
+ {% set statements = [] %}
 
- {% set sql %}
- insert into {{ this.render() }} ({{ cols_sql }}) values
- {% for row in chunk -%}
- ({%- for column in agate_table.column_names -%}
- %s
- {%- if not loop.last%},{%- endif %}
- {%- endfor -%})
- {%- if not loop.last%},{%- endif %}
- {%- endfor %}
- {% endset %}
+ {% for chunk in agate_table.rows | batch(batch_size) %}
+ {% set bindings = [] %}
 
- {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+ {% for row in chunk %}
+ {% do bindings.extend(row) %}
+ {% endfor %}
 
- {% if loop.index0 == 0 %}
- {% do statements.append(sql) %}
- {% endif %}
- {% endfor %}
+ {% set sql %}
+ insert into {{ this.render() }} ({{ cols_sql }}) values
+ {% for row in chunk -%}
+ ({%- for column in agate_table.column_names -%}
+ {{ get_binding_char() }}
+ {%- if not loop.last%},{%- endif %}
+ {%- endfor -%})
+ {%- if not loop.last%},{%- endif %}
+ {%- endfor %}
+ {% endset %}
 
- {# Return SQL so we can render it out into the compiled files #}
- {{ return(statements[0]) }}
-{% endmacro %}
+ {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
 
+ {% if loop.index0 == 0 %}
+ {% do statements.append(sql) %}
+ {% endif %}
+ {% endfor %}
 
-{% macro default__load_csv_rows(model, agate_table) %}
- {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}
+ {# Return SQL so we can render it out into the compiled files #}
+ {{ return(statements[0]) }}
 {% endmacro %}
 
-
 {% materialization seed, default %}
 
  {%- set identifier = model['alias'] -%}