From 2afc2026783425fbe1a3405d8913d2a50462bc46 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sat, 2 Dec 2017 17:21:20 -0500 Subject: [PATCH] Make run_started_at timezone aware (#553) (#556) * Make run_started_at timezone aware Set run_started_at timezone to UTC Enable timezone change in models Extend requirements Extend tests * Address comments from code review Create modules namespace to context Move pytz to modules Add new dependencies to setup.py --- dbt/context/common.py | 4 ++ dbt/tracking.py | 3 +- dev_requirements.txt | 2 + requirements.txt | 2 + setup.py | 2 + .../025_timezones_test/models/timezones.sql | 10 ++++ .../025_timezones_test/test_timezones.py | 55 +++++++++++++++++++ 7 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test/integration/025_timezones_test/models/timezones.sql create mode 100644 test/integration/025_timezones_test/test_timezones.py diff --git a/dbt/context/common.py b/dbt/context/common.py index 834e3cc3d19..072cb62a7e4 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -1,5 +1,6 @@ import json import os +import pytz import voluptuous from dbt.adapters.factory import get_adapter @@ -288,6 +289,9 @@ def generate(model, project, flat_graph, provider=None): "graph": flat_graph, "log": log, "model": model, + "modules": { + "pytz": pytz, + }, "post_hooks": post_hooks, "pre_hooks": pre_hooks, "ref": provider.ref(model, project, profile, flat_graph), diff --git a/dbt/tracking.py b/dbt/tracking.py index 39e5b7b0436..0a724e878a9 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -4,6 +4,7 @@ from snowplow_tracker import SelfDescribingJson, disable_contracts from datetime import datetime +import pytz import platform import uuid import yaml @@ -44,7 +45,7 @@ def __init__(self): self.id = None self.invocation_id = str(uuid.uuid4()) - self.run_started_at = datetime.now() + self.run_started_at = datetime.now(tz=pytz.utc) def state(self): return "do not track" if self.do_not_track else "tracking" diff --git a/dev_requirements.txt b/dev_requirements.txt index bd576fdfb5c..d205dbd8f9d 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,8 @@ +freezegun==0.3.9 nose>=1.3.7 mock>=1.3.0 pep8>=1.6.2 +pytz==2017.2 bumpversion==0.5.3 coverage==4.2 tox==2.5.0 diff --git a/requirements.txt b/requirements.txt index 262b002e96c..d5947c66764 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ argparse>=1.2.1 +freezegun==0.3.9 Jinja2>=2.8 +pytz==2017.2 PyYAML>=3.11 psycopg2==2.7.1 sqlparse==0.2.3 diff --git a/setup.py b/setup.py index 2942c39b785..9c9421c79b2 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,9 @@ 'scripts/dbt', ], install_requires=[ + 'freezegun==0.3.9', 'Jinja2>=2.8', + 'pytz==2017.2', 'PyYAML>=3.11', 'psycopg2==2.7.1', 'sqlparse==0.2.3', diff --git a/test/integration/025_timezones_test/models/timezones.sql b/test/integration/025_timezones_test/models/timezones.sql new file mode 100644 index 00000000000..87d565487e1 --- /dev/null +++ b/test/integration/025_timezones_test/models/timezones.sql @@ -0,0 +1,10 @@ + +{{ + config( + materialized='table' + ) +}} + +select + '{{ run_started_at.astimezone(modules.pytz.timezone("America/New_York")) }}' as run_started_at_est, + '{{ run_started_at }}' as run_started_at_utc diff --git a/test/integration/025_timezones_test/test_timezones.py b/test/integration/025_timezones_test/test_timezones.py new file mode 100644 index 00000000000..af995eaef31 --- /dev/null +++ b/test/integration/025_timezones_test/test_timezones.py @@ -0,0 +1,55 @@ +from freezegun import freeze_time +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest + + +class TestTimezones(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + + @property + def schema(self): + return "timezones_025" + + @property + def models(self): + return "test/integration/025_timezones_test/models" + + @property + def profile_config(self): + return { + 'test': { + 'outputs': { + 'dev': { + 'type': 'postgres', + 'threads': 1, + 'host': 'database', + 'port': 5432, + 'user': "root", + 'pass': "password", + 'dbname': 'dbt', + 'schema': self.unique_schema() + }, + }, + 'target': 'dev' + } + } + + @property + def query(self): + return """ + select + run_started_at_est, + run_started_at_utc + from {schema}.timezones + """.format(schema=self.unique_schema()) + + @freeze_time("2017-01-01 03:00:00", tz_offset=0) + @attr(type='postgres') + def test_run_started_at(self): + self.run_dbt(['run']) + result = self.run_sql(self.query, fetch='all')[0] + est, utc = result + self.assertEqual(utc, '2017-01-01 03:00:00+00:00') + self.assertEqual(est, '2016-12-31 22:00:00-05:00')