Skip to content

Commit

Permalink
add back dynamic batch sizing
Browse files Browse the repository at this point in the history
  • Loading branch information
dataders committed Nov 5, 2021
1 parent d62dc57 commit fb6abed
Showing 1 changed file with 45 additions and 0 deletions.
45 changes: 45 additions & 0 deletions dbt/include/sqlserver/macros/materializations/seed/seed.sql
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,49 @@

{% macro sqlserver__get_batch_size() %}
{{ return(400) }}
{% endmacro %}

{% macro sqlserver__load_csv_rows(model, agate_table) %}
{% set max_batch_size = get_batch_size() %}
{% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
{% set batch_size = calc_batch_size(cols_sql|length, max_batch_size) %}

{{ return(basic_load_csv_rows(model, batch_size, agate_table) )}}
{% endmacro %}


{% macro basic_load_csv_rows(model, batch_size, agate_table) %}

{% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
{% set bindings = [] %}

{% set statements = [] %}

{% for chunk in agate_table.rows | batch(batch_size) %}
{% set bindings = [] %}

{% for row in chunk %}
{% do bindings.extend(row) %}
{% endfor %}

{% set sql %}
insert into {{ this.render() }} ({{ cols_sql }}) values
{% for row in chunk -%}
({%- for column in agate_table.column_names -%}
{{ get_binding_char() }}
{%- if not loop.last%},{%- endif %}
{%- endfor -%})
{%- if not loop.last%},{%- endif %}
{%- endfor %}
{% endset %}

{% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}

{% if loop.index0 == 0 %}
{% do statements.append(sql) %}
{% endif %}
{% endfor %}

{# Return SQL so we can render it out into the compiled files #}
{{ return(statements[0]) }}
{% endmacro %}

0 comments on commit fb6abed

Please sign in to comment.