Skip to content

Commit

Permalink
fix snowflake seed types, add chunking for seed files with more than …
Browse files Browse the repository at this point in the history
…16k rows (#694)
  • Loading branch information
cmcarthur authored Apr 9, 2018
1 parent 68634a2 commit 848ff6a
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 13 deletions.
31 changes: 18 additions & 13 deletions dbt/adapters/postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import dbt.exceptions
import agate

from dbt.utils import chunks
from dbt.logger import GLOBAL_LOGGER as logger


Expand Down Expand Up @@ -174,7 +175,7 @@ def convert_text_type(cls, agate_table, col_idx):
@classmethod
def convert_number_type(cls, agate_table, col_idx):
decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
return "numeric" if decimals else "integer"
return "float8" if decimals else "integer"

@classmethod
def convert_boolean_type(cls, agate_table, col_idx):
Expand Down Expand Up @@ -213,19 +214,23 @@ def reset_csv_table(cls, profile, schema, table_name, agate_table,

@classmethod
def load_csv_rows(cls, profile, schema, table_name, agate_table):
bindings = []
placeholders = []
cols_sql = ", ".join(c for c in agate_table.column_names)

for row in agate_table.rows:
bindings += row
placeholders.append("({})".format(
", ".join("%s" for _ in agate_table.column_names)))
for chunk in chunks(agate_table.rows, 10000):
bindings = []
placeholders = []

sql = ('insert into {}.{} ({}) values {}'
.format(cls.quote(schema),
cls.quote(table_name),
cols_sql,
",\n".join(placeholders)))
for row in chunk:
bindings += row
placeholders.append("({})".format(
", ".join("%s" for _ in agate_table.column_names)))

cls.add_query(profile, sql, bindings=bindings, abridge_sql_log=True)
sql = ('insert into {}.{} ({}) values {}'
.format(cls.quote(schema),
cls.quote(table_name),
cols_sql,
",\n".join(placeholders)))

cls.add_query(profile, sql,
bindings=bindings,
abridge_sql_log=True)
6 changes: 6 additions & 0 deletions dbt/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@ def coalesce(*args):
return None


def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]


def get_profile_from_project(project):
target_name = project.get('target', {})
profile = project.get('outputs', {}).get(target_name, {})
Expand Down

0 comments on commit 848ff6a

Please sign in to comment.