From d0d7ef2ccbae609e3b11e7481f5c3fdfad6aae9b Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sun, 4 Dec 2016 14:34:19 -0500 Subject: [PATCH 1/3] track dbt env --- dbt/tracking.py | 19 ++++++++++++++++--- .../invocation_env_context.json | 10 ++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 events/schemas/com.fishtownanalytics/invocation_env_context.json diff --git a/dbt/tracking.py b/dbt/tracking.py index 515e2bcbc20..35e416ac88d 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -23,6 +23,9 @@ INVOCATION_SPEC = "https://raw.githubusercontent.com/analyst-collective/dbt/master/events/schemas/com.fishtownanalytics/invocation_event.json" PLATFORM_SPEC = "https://raw.githubusercontent.com/analyst-collective/dbt/master/events/schemas/com.fishtownanalytics/platform_context.json" RUN_MODEL_SPEC = "https://raw.githubusercontent.com/analyst-collective/dbt/master/events/schemas/com.fishtownanalytics/run_model_context.json" +INVOCATION_ENV_SPEC = "https://raw.githubusercontent.com/analyst-collective/dbt/feature/track_dbt_environment/events/schemas/com.fishtownanalytics/invocation_env_context.json" + +DBT_INVOCATION_ENV = 'DBT_INVOCATION_ENV' emitter = Emitter(COLLECTOR_URL, protocol=COLLECTOR_PROTOCOL, buffer_size=1) tracker = Tracker(emitter, namespace="cf", app_id="dbt") @@ -124,8 +127,17 @@ def get_platform_context(): return SelfDescribingJson(PLATFORM_SPEC, data) +def get_dbt_env_context(): + dbt_invocation_env = os.getenv(DBT_INVOCATION_ENV, 'manual') + data = { + "environment" : dbt_invocation_env, + } + + return SelfDescribingJson(INVOCATION_ENV_SPEC, data) + invocation_id = str(uuid.uuid4()) platform_context = get_platform_context() +env_context = get_dbt_env_context() user = get_user() subject = Subject() @@ -146,7 +158,8 @@ def track(*args, **kwargs): def track_invocation_start(project=None, args=None): invocation_context = get_invocation_start_context(invocation_id, user, project, args) - context = [invocation_context, platform_context] + context = [invocation_context, platform_context, env_context] + import ipdb; ipdb.set_trace() track(category="dbt", action='invocation', label='start', context=context) def track_model_run(options): @@ -156,12 +169,12 @@ def track_model_run(options): def track_invocation_end(project=None, args=None, result_type=None, result=None): invocation_context = get_invocation_end_context(invocation_id, user, project, args, result_type, result) - context = [invocation_context, platform_context] + context = [invocation_context, platform_context, env_context] track(category="dbt", action='invocation', label='end', context=context) def track_invalid_invocation(project=None, args=None, result_type=None, result=None): invocation_context = get_invocation_invalid_context(invocation_id, user, project, args, result_type, result) - context = [invocation_context, platform_context] + context = [invocation_context, platform_context, env_context] track(category="dbt", action='invocation', label='invalid', context=context) def flush(): diff --git a/events/schemas/com.fishtownanalytics/invocation_env_context.json b/events/schemas/com.fishtownanalytics/invocation_env_context.json new file mode 100644 index 00000000000..4f98e3f7f4e --- /dev/null +++ b/events/schemas/com.fishtownanalytics/invocation_env_context.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "title": "invocation_env", + "description": "DBT invocation environment type", + "properties": { + "environment": { + "type": "string" + } + } +} From af85f952cd493850d28fe88721e3f56cfb6caae2 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sun, 4 Dec 2016 15:27:32 -0500 Subject: [PATCH 2/3] default to 'manual' --- dbt/tracking.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbt/tracking.py b/dbt/tracking.py index 35e416ac88d..da3c592e120 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -128,7 +128,12 @@ def get_platform_context(): return SelfDescribingJson(PLATFORM_SPEC, data) def get_dbt_env_context(): - dbt_invocation_env = os.getenv(DBT_INVOCATION_ENV, 'manual') + default = 'manual' + + dbt_invocation_env = os.getenv(DBT_INVOCATION_ENV, default) + if dbt_invocation_env == '': + dbt_invocation_env = default + data = { "environment" : dbt_invocation_env, } @@ -159,7 +164,6 @@ def track(*args, **kwargs): def track_invocation_start(project=None, args=None): invocation_context = get_invocation_start_context(invocation_id, user, project, args) context = [invocation_context, platform_context, env_context] - import ipdb; ipdb.set_trace() track(category="dbt", action='invocation', label='start', context=context) def track_model_run(options): From c8ba6e8d6f565b42cc098a2d2a01c022b4dcd25b Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Sun, 4 Dec 2016 20:43:35 -0500 Subject: [PATCH 3/3] add dbt run context env vars for integrations --- test/runner.sh | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/runner.sh b/test/runner.sh index 9b707932fcf..c11905771fd 100755 --- a/test/runner.sh +++ b/test/runner.sh @@ -10,5 +10,5 @@ if [ $# = 0 ]; then tox else echo "Running specified tests" - nosetests -v --nocapture --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov $@ + DBT_INVOCATION_ENV="ci-local" nosetests -v --nocapture --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov $@ fi diff --git a/tox.ini b/tox.ini index 20607251b90..81e65b8b070 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ envlist = py27, py35 [testenv] -commands = /bin/bash -c 'HOME=/root/ {envpython} $(which nosetests) -v --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/unit test/integration/*' +commands = /bin/bash -c 'HOME=/root/ DBT_INVOCATION_ENV=ci-circle {envpython} $(which nosetests) -v --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/unit test/integration/*' deps = -rrequirements.txt -rdev_requirements.txt @@ -17,6 +17,7 @@ deps = basepython = {env:PYTHON:}\python.exe setenv = DBT_CONFIG_DIR = ~/.dbt + DBT_INVOCATION_ENV = ci-appveyor commands = nosetests -v --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/unit test/integration/ deps = -rrequirements.txt