-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
run hooks outside of a transaction #510
Changes from 4 commits
b9930c7
0e60445
7f61366
127563b
f250d1f
6dd7c3e
f681f77
2c20689
022b3d7
98e4227
df98bcf
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,10 +17,16 @@ | |
def get_hooks(model, context, hook_key): | ||
hooks = model.get('config', {}).get(hook_key, []) | ||
|
||
if isinstance(hooks, basestring): | ||
if not isinstance(hooks, (list, tuple)): | ||
hooks = [hooks] | ||
|
||
return hooks | ||
wrapped_hooks = [] | ||
for hook in hooks: | ||
if isinstance(hook, dict): | ||
hook = json.dumps(hook) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is the only part of this branch that doesn't feel right to me. it seems like we should do the opposite: parse all the hooks to their dictionary version and return those. then you don't need to decode each hook individually in a loop before you can perform filter logic. that would let you write, for example:
(and, in general, lets you do more powerful comprehensions on hooks) |
||
wrapped_hooks.append(hook) | ||
|
||
return wrapped_hooks | ||
|
||
|
||
class DatabaseWrapper(object): | ||
|
@@ -227,6 +233,15 @@ def fn(string): | |
return fn | ||
|
||
|
||
def fromjson(node): | ||
def fn(string, default=None): | ||
try: | ||
return json.loads(string) | ||
except ValueError as e: | ||
return default | ||
return fn | ||
|
||
|
||
def generate(model, project, flat_graph, provider=None): | ||
""" | ||
Not meant to be called directly. Call with either: | ||
|
@@ -270,6 +285,7 @@ def generate(model, project, flat_graph, provider=None): | |
"schema": schema, | ||
"sql": model.get('injected_sql'), | ||
"sql_now": adapter.date_function(), | ||
"fromjson": fromjson(model), | ||
"target": target, | ||
"this": dbt.utils.This( | ||
schema, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,14 @@ | ||
{% macro run_hooks(hooks) %} | ||
{% macro run_hooks(hooks, inside_transaction=True) %} | ||
{% for hook in hooks %} | ||
{% call statement() %} | ||
{{ hook }}; | ||
{% endcall %} | ||
{%- set hook_data = fromjson(render(hook), {}) -%} | ||
{%- set hook_is_in_transaction = hook_data.get('transaction', True) -%}; | ||
{%- set hook_sql = hook_data.get('sql', hook) -%}; | ||
|
||
{%- if hook_is_in_transaction == inside_transaction -%} | ||
{% call statement(auto_begin=inside_transaction) %} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is slick |
||
{{ hook_sql }} | ||
{% endcall %} | ||
{%- endif -%} | ||
{% endfor %} | ||
{% endmacro %} | ||
|
||
|
@@ -21,6 +27,26 @@ | |
{% endmacro %} | ||
|
||
|
||
{% macro make_hook_config(sql, inside_transaction) %} | ||
{{ {"sql": sql, "transaction": inside_transaction} | tojson }} | ||
{% endmacro %} | ||
|
||
|
||
{% macro before_begin(sql) %} | ||
{{ make_hook_config(sql, inside_transaction=False) }} | ||
{% endmacro %} | ||
|
||
|
||
{% macro after_commit(sql) %} | ||
{{ make_hook_config(sql, inside_transaction=False) }} | ||
{% endmacro %} | ||
|
||
|
||
{% macro vacuum(tbl) %} | ||
{{ after_commit('vacuum ' ~ adapter.quote_schema_and_table(tbl.schema, tbl.name)) }} | ||
{% endmacro %} | ||
|
||
|
||
{% macro drop_if_exists(existing, name) %} | ||
{% set existing_type = existing.get(name) %} | ||
{% if existing_type is not none %} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -91,22 +91,13 @@ def update_in_model_config(self, config): | |
self.in_model_config.update(config) | ||
|
||
def __get_hooks(self, relevant_configs, key): | ||
hooks = [] | ||
|
||
if key not in relevant_configs: | ||
return [] | ||
|
||
new_hooks = relevant_configs[key] | ||
if type(new_hooks) not in [list, tuple]: | ||
new_hooks = [new_hooks] | ||
|
||
for hook in new_hooks: | ||
if not isinstance(hook, basestring): | ||
name = ".".join(self.fqn) | ||
dbt.exceptions.raise_compiler_error( | ||
"{} for model {} is not a string!".format(key, name)) | ||
hooks = relevant_configs[key] | ||
if not isinstance(hooks, (list, tuple)): | ||
hooks = [hooks] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nice |
||
|
||
hooks.append(hook) | ||
return hooks | ||
|
||
def smart_update(self, mutable_config, new_configs): | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
does this automatically roll back open transactions? would want to be sure we don't create a deadlock because of an open tx
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yeah,
release_connection
callsrollback
under the hood if the transaction is open. This change is to fix an issue where we'd try to rollback transactions that weren't open