From f223eb8fd81f8b8ba8054447c4ea15e332b5e496 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 17:41:23 -0700 Subject: [PATCH 01/17] please work --- .github/workflows/dbt_deploy.yml | 4 ++-- .github/workflows/dbt_staging.yml | 4 ++-- .gitignore | 4 +++- README.md | 4 ++-- dbt_project.yml | 2 +- profiles.yml | 8 ++++---- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index 0e8bead..83e84b6 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -46,7 +46,7 @@ jobs: - name: submit artifacts to datafold run: | set -ex - datafold dbt upload --ci-config-id 345 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + datafold dbt upload --ci-config-id 8350 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ 'production' }}" @@ -57,7 +57,7 @@ jobs: # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - name: Upload manifest to S3 run: | - aws s3 cp target/manifest.json s3://vhol-datafold-dbt-prod-manifest/manifest.json + aws s3 cp target/manifest.json s3://vhol-prod-manifest-demo/manifest.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/dbt_staging.yml b/.github/workflows/dbt_staging.yml index e4ff4bd..f1a4a60 100644 --- a/.github/workflows/dbt_staging.yml +++ b/.github/workflows/dbt_staging.yml @@ -40,7 +40,7 @@ jobs: # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - name: Grab production manifest from S3 run: | - aws s3 cp s3://vhol-datafold-dbt-prod-manifest/manifest.json ./manifest.json + aws s3 cp s3://vhol-prod-manifest-demo/manifest.json ./manifest.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -60,7 +60,7 @@ jobs: - name: submit artifacts to datafold run: | set -ex - datafold dbt upload --ci-config-id 345 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + datafold dbt upload --ci-config-id 8350 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}" diff --git a/.gitignore b/.gitignore index b235435..dee9201 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,6 @@ cython_debug/ target/ dbt_packages/ -logs/ \ No newline at end of file +logs/ + +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 9916bad..0d8822d 100644 --- a/README.md +++ b/README.md @@ -49,13 +49,13 @@ source venv/bin/activate "Sid": "ListObjectsInBucket", "Effect": "Allow", "Action": ["s3:ListBucket"], - "Resource": ["arn:aws:s3:::vhol-datafold-dbt-prod-manifest"] # TODO: replace with your own bucket name + "Resource": ["arn:aws:s3:::vhol-prod-manifest-demo"] # TODO: replace with your own bucket name }, { "Sid": "AllObjectActions", "Effect": "Allow", "Action": "s3:*Object", - "Resource": ["arn:aws:s3:::vhol-datafold-dbt-prod-manifest/*"] # TODO: replace with your own bucket name + "Resource": ["arn:aws:s3:::vhol-prod-manifest-demo/*"] # TODO: replace with your own bucket name } ] } diff --git a/dbt_project.yml b/dbt_project.yml index 595567e..f38f54f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -29,7 +29,7 @@ vars: data_diff: prod_database: DEMO prod_schema: CORE - datasource_id: 4932 + datasource_id: 8350 # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models diff --git a/profiles.yml b/profiles.yml index 1871dfa..ff615c4 100644 --- a/profiles.yml +++ b/profiles.yml @@ -7,9 +7,9 @@ vhol_demo: account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - role: DEMO_ROLE + role: DATAFOLDROLE database: DEMO - warehouse: INTEGRATION + warehouse: COMPUTE_WH schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" threads: 24 @@ -18,8 +18,8 @@ vhol_demo: account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - role: DEMO_ROLE + role: DATAFOLDROLE database: DEMO - warehouse: INTEGRATION + warehouse: COMPUTE_WH schema: CORE threads: 24 \ No newline at end of file From 1777f4aff1b0fcdda7784f9b67cd83dfe86182ab Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 20:58:15 -0700 Subject: [PATCH 02/17] Feature/sql change (#1) * made changes to dbt models and ran dbt build * Making minor change for rerun * dummy commit to trigger pr * Added protobuf to requirements to resolve error * dummy commit to trigger push after updating snowflake permissions * corrected datafold ci-config-id * Corrected datfold ci-config-id in project yml * :) * Attempting another PR after regenerating datafold api key --- .github/workflows/dbt_deploy.yml | 2 +- .github/workflows/dbt_staging.yml | 2 +- README.md | 4 +++- models/core/dim_orgs.sql | 13 +++++++------ models/syncs/sales_sync.sql | 2 +- requirements.txt | 3 ++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index 83e84b6..fbdb3bd 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -46,7 +46,7 @@ jobs: - name: submit artifacts to datafold run: | set -ex - datafold dbt upload --ci-config-id 8350 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + datafold dbt upload --ci-config-id 421 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ 'production' }}" diff --git a/.github/workflows/dbt_staging.yml b/.github/workflows/dbt_staging.yml index f1a4a60..2508806 100644 --- a/.github/workflows/dbt_staging.yml +++ b/.github/workflows/dbt_staging.yml @@ -60,7 +60,7 @@ jobs: - name: submit artifacts to datafold run: | set -ex - datafold dbt upload --ci-config-id 8350 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + datafold dbt upload --ci-config-id 421 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}" diff --git a/README.md b/README.md index 0d8822d..996ddb6 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ Please replace the placeholder with your own bucket name for the policy to work 3. Set up your GitHub secrets +3a. Install protobuf: python3 -m pip install protobuf==4.25.3 +3a. Install dbt-snowflake adapter 4. Run the below to build your dbt project on the `main` branch @@ -88,7 +90,7 @@ dbt build --target prod 6. Run the below to switch branches and test your CI pipelines ```bash -git checkout -b feature/your-feature +git checkout -b feature/sql-change ``` 7. Make some changes to your dbt models and push to your remote repository. You can copy and paste the examples here and run it locally: [example_dbt_changes](example_dbt_changes/) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 73beb44..5d2262d 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,9 +2,10 @@ WITH orgs AS ( --prod SELECT org_id - , MIN(event_timestamp) AS created_at - FROM {{ ref('signed_in') }} - GROUP BY 1 + , org_name + , employee_range + , created_at + FROM {{ ref('org_created') }} -- --dev -- SELECT @@ -28,7 +29,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , price as sub_price + , coalesce(price, 0) as sub_price FROM {{ ref('subscription_created') }} ) @@ -38,8 +39,8 @@ SELECT , created_at , num_users , sub_created_at - , sub_plan + , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index ff10d8c..2567aac 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL + WHERE sub_plan IS NULL or sub_plan = 'Individual' ) , final AS ( diff --git a/requirements.txt b/requirements.txt index 9ffd3fb..655492f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ dbt-core==1.7.7 dbt-snowflake==1.7.1 data-diff==0.11.0 -datafold-sdk==0.0.19 \ No newline at end of file +datafold-sdk==0.0.19 +protobuf==4.25.3 \ No newline at end of file From 1c28017ef9528531b6d25b522fe9b65d5a753943 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 21:19:57 -0700 Subject: [PATCH 03/17] Update dbt_deploy.yml --- .github/workflows/dbt_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index fbdb3bd..fcc9175 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -27,7 +27,7 @@ jobs: # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - name: Grab production manifest from S3 run: | - aws s3 cp s3://vhol-datafold-dbt-prod-manifest/manifest.json ./manifest.json + aws s3 cp s3://vhol-prod-manifest-demo/manifest.json ./manifest.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 2764479a1f4323a5a102f03618571abb08445808 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 21:43:40 -0700 Subject: [PATCH 04/17] Datafold DataDiffing (#2) * made changes to dbt models and ran dbt build * Making minor change for rerun * dummy commit to trigger pr * Added protobuf to requirements to resolve error * dummy commit to trigger push after updating snowflake permissions * corrected datafold ci-config-id * Corrected datfold ci-config-id in project yml * :) * Attempting another PR after regenerating datafold api key * Corrected s3 bucket name in dbt_deploy hml * making changes to dbt models for data diff --- models/core/dim_orgs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 5d2262d..73e50e2 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -43,4 +43,4 @@ SELECT , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) \ No newline at end of file From 427e23c8390a634ac6f9bd47c484b90ae5ad1b22 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 21:55:54 -0700 Subject: [PATCH 05/17] Datadiff after reuploading manifest (#3) * made changes to models for datadiff demo * Triggering PR after reuploading manifest --- README.md | 2 +- models/core/dim_orgs.sql | 2 +- models/syncs/sales_sync.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 996ddb6..5a7414b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ You should be able to run this self-contained example, use the `yml` workflow fi If you have questions unique to your tech stack, schedule a call with us at: [Datafold](https://www.datafold.com/) -## What does success look like? +## What does success look like?? - [ ] Slim CI pipeline that runs and tests only the models that have changed and their downstream models in your pull requests - [ ] Automated Datafold data diffs in your pull requests if you have a Datafold account diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 73e50e2..5d2262d 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -43,4 +43,4 @@ SELECT , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) \ No newline at end of file +LEFT JOIN subscriptions USING (org_id) diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index 2567aac..f98d35d 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -18,4 +18,4 @@ WITH org_events AS ( GROUP BY 1 ) -SELECT * FROM final \ No newline at end of file +SELECT * FROM final From 154f6e9dde7b9be6454f110cc64a668dccd09af2 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:01:47 -0700 Subject: [PATCH 06/17] Datadiff ??? (#4) * made changes to models for datadiff demo * Triggering PR after reuploading manifest From c2ba5fe40ec9a26de9775e26b32c438c8cb0fa8b Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:06:11 -0700 Subject: [PATCH 07/17] revert models code to original (will attempt to re-do changes from example_dbt_changes) --- models/core/dim_orgs.sql | 13 ++++++------- models/syncs/sales_sync.sql | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 5d2262d..73beb44 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,10 +2,9 @@ WITH orgs AS ( --prod SELECT org_id - , org_name - , employee_range - , created_at - FROM {{ ref('org_created') }} + , MIN(event_timestamp) AS created_at + FROM {{ ref('signed_in') }} + GROUP BY 1 -- --dev -- SELECT @@ -29,7 +28,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , coalesce(price, 0) as sub_price + , price as sub_price FROM {{ ref('subscription_created') }} ) @@ -39,8 +38,8 @@ SELECT , created_at , num_users , sub_created_at - , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan + , sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index f98d35d..ff10d8c 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL or sub_plan = 'Individual' + WHERE sub_plan IS NULL ) , final AS ( @@ -18,4 +18,4 @@ WITH org_events AS ( GROUP BY 1 ) -SELECT * FROM final +SELECT * FROM final \ No newline at end of file From 78afdc12efb261a72ea4b01d1d4c4c4ffa837c4d Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:09:35 -0700 Subject: [PATCH 08/17] undo revert, keep changes --- models/core/dim_orgs.sql | 13 +++++++------ models/syncs/sales_sync.sql | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 73beb44..5d2262d 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,9 +2,10 @@ WITH orgs AS ( --prod SELECT org_id - , MIN(event_timestamp) AS created_at - FROM {{ ref('signed_in') }} - GROUP BY 1 + , org_name + , employee_range + , created_at + FROM {{ ref('org_created') }} -- --dev -- SELECT @@ -28,7 +29,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , price as sub_price + , coalesce(price, 0) as sub_price FROM {{ ref('subscription_created') }} ) @@ -38,8 +39,8 @@ SELECT , created_at , num_users , sub_created_at - , sub_plan + , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index ff10d8c..f98d35d 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL + WHERE sub_plan IS NULL or sub_plan = 'Individual' ) , final AS ( @@ -18,4 +18,4 @@ WITH org_events AS ( GROUP BY 1 ) -SELECT * FROM final \ No newline at end of file +SELECT * FROM final From 3527c2c232797cdf35615c77eac55c8717882d0a Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:16:32 -0700 Subject: [PATCH 09/17] made changes to models and dbt build dev --- models/core/dim_orgs.sql | 13 ++++++------- models/syncs/sales_sync.sql | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 5d2262d..0b664ba 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,10 +2,9 @@ WITH orgs AS ( --prod SELECT org_id - , org_name - , employee_range - , created_at - FROM {{ ref('org_created') }} + , MIN(event_timestamp) AS created_at + FROM {{ ref('signed_in') }} + GROUP BY 1 -- --dev -- SELECT @@ -29,7 +28,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , coalesce(price, 0) as sub_price + , price as sub_price FROM {{ ref('subscription_created') }} ) @@ -39,8 +38,8 @@ SELECT , created_at , num_users , sub_created_at - , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan + , sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) \ No newline at end of file diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index f98d35d..538bbcb 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL or sub_plan = 'Individual' + WHERE sub_plan IS NULL ) , final AS ( From 9472aa4a6f1ca5f1ed2fa035422bbd2c0b28b31c Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:25:16 -0700 Subject: [PATCH 10/17] changing models to see differences in datafold woohoo git push --- models/core/dim_orgs.sql | 13 +++++++------ models/syncs/sales_sync.sql | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 0b664ba..5d2262d 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,9 +2,10 @@ WITH orgs AS ( --prod SELECT org_id - , MIN(event_timestamp) AS created_at - FROM {{ ref('signed_in') }} - GROUP BY 1 + , org_name + , employee_range + , created_at + FROM {{ ref('org_created') }} -- --dev -- SELECT @@ -28,7 +29,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , price as sub_price + , coalesce(price, 0) as sub_price FROM {{ ref('subscription_created') }} ) @@ -38,8 +39,8 @@ SELECT , created_at , num_users , sub_created_at - , sub_plan + , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) \ No newline at end of file +LEFT JOIN subscriptions USING (org_id) diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index 538bbcb..2567aac 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL + WHERE sub_plan IS NULL or sub_plan = 'Individual' ) , final AS ( @@ -18,4 +18,4 @@ WITH org_events AS ( GROUP BY 1 ) -SELECT * FROM final +SELECT * FROM final \ No newline at end of file From 390c371bc9cf6fff0681e52d13aeb58d069877f8 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:30:59 -0700 Subject: [PATCH 11/17] asdgasdg --- models/core/dim_orgs.sql | 13 ++++++------- models/syncs/sales_sync.sql | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 5d2262d..0b664ba 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,10 +2,9 @@ WITH orgs AS ( --prod SELECT org_id - , org_name - , employee_range - , created_at - FROM {{ ref('org_created') }} + , MIN(event_timestamp) AS created_at + FROM {{ ref('signed_in') }} + GROUP BY 1 -- --dev -- SELECT @@ -29,7 +28,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , coalesce(price, 0) as sub_price + , price as sub_price FROM {{ ref('subscription_created') }} ) @@ -39,8 +38,8 @@ SELECT , created_at , num_users , sub_created_at - , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan + , sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) \ No newline at end of file diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index 2567aac..ff10d8c 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL or sub_plan = 'Individual' + WHERE sub_plan IS NULL ) , final AS ( From 296dd6c10799af4d48f670ff6b54842920596990 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:36:43 -0700 Subject: [PATCH 12/17] making changes to dbt models to see differences --- models/core/dim_orgs.sql | 13 +++++++------ models/syncs/sales_sync.sql | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 0b664ba..5d2262d 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,9 +2,10 @@ WITH orgs AS ( --prod SELECT org_id - , MIN(event_timestamp) AS created_at - FROM {{ ref('signed_in') }} - GROUP BY 1 + , org_name + , employee_range + , created_at + FROM {{ ref('org_created') }} -- --dev -- SELECT @@ -28,7 +29,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , price as sub_price + , coalesce(price, 0) as sub_price FROM {{ ref('subscription_created') }} ) @@ -38,8 +39,8 @@ SELECT , created_at , num_users , sub_created_at - , sub_plan + , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) \ No newline at end of file +LEFT JOIN subscriptions USING (org_id) diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index ff10d8c..2567aac 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL + WHERE sub_plan IS NULL or sub_plan = 'Individual' ) , final AS ( From 8027c7684eb768377c893fd8000e680a4c4e7925 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:56:54 -0700 Subject: [PATCH 13/17] commit w/o performing dbt build beforehand --- .github/workflows/dbt_deploy.yml | 4 ++-- models/core/dim_orgs.sql | 13 ++++++------- models/syncs/sales_sync.sql | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index fcc9175..f2d78ac 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -31,7 +31,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: us-east-2 - name: dbt build run: dbt build --target prod --select state:modified+ --defer --state ./ --exclude config.materialized:snapshot --full-refresh --profiles-dir ./ @@ -61,4 +61,4 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: us-east-2 diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 5d2262d..0b664ba 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,10 +2,9 @@ WITH orgs AS ( --prod SELECT org_id - , org_name - , employee_range - , created_at - FROM {{ ref('org_created') }} + , MIN(event_timestamp) AS created_at + FROM {{ ref('signed_in') }} + GROUP BY 1 -- --dev -- SELECT @@ -29,7 +28,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , coalesce(price, 0) as sub_price + , price as sub_price FROM {{ ref('subscription_created') }} ) @@ -39,8 +38,8 @@ SELECT , created_at , num_users , sub_created_at - , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan + , sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) \ No newline at end of file diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index 2567aac..ff10d8c 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL or sub_plan = 'Individual' + WHERE sub_plan IS NULL ) , final AS ( From 901bc5e6ead5cbd26a66e669ecdcf3dbca36c25d Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Tue, 16 Jul 2024 22:58:51 -0700 Subject: [PATCH 14/17] Feature/dbt build (#5) * revert models code to original (will attempt to re-do changes from example_dbt_changes) * undo revert, keep changes * made changes to models and dbt build dev * changing models to see differences in datafold woohoo git push * asdgasdg * making changes to dbt models to see differences * commit w/o performing dbt build beforehand --- .github/workflows/dbt_deploy.yml | 4 ++-- models/core/dim_orgs.sql | 13 ++++++------- models/syncs/sales_sync.sql | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index fcc9175..f2d78ac 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -31,7 +31,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: us-east-2 - name: dbt build run: dbt build --target prod --select state:modified+ --defer --state ./ --exclude config.materialized:snapshot --full-refresh --profiles-dir ./ @@ -61,4 +61,4 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: us-east-2 diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 5d2262d..0b664ba 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -2,10 +2,9 @@ WITH orgs AS ( --prod SELECT org_id - , org_name - , employee_range - , created_at - FROM {{ ref('org_created') }} + , MIN(event_timestamp) AS created_at + FROM {{ ref('signed_in') }} + GROUP BY 1 -- --dev -- SELECT @@ -29,7 +28,7 @@ WITH orgs AS ( org_id , event_timestamp AS sub_created_at , plan as sub_plan - , coalesce(price, 0) as sub_price + , price as sub_price FROM {{ ref('subscription_created') }} ) @@ -39,8 +38,8 @@ SELECT , created_at , num_users , sub_created_at - , case when num_users = 1 then 'Individual' else sub_plan end as sub_plan + , sub_plan , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) +LEFT JOIN subscriptions USING (org_id) \ No newline at end of file diff --git a/models/syncs/sales_sync.sql b/models/syncs/sales_sync.sql index f98d35d..ff10d8c 100644 --- a/models/syncs/sales_sync.sql +++ b/models/syncs/sales_sync.sql @@ -3,7 +3,7 @@ WITH org_events AS ( * FROM {{ ref('dim_orgs') }} LEFT JOIN {{ ref('feature_used') }} USING (org_id) - WHERE sub_plan IS NULL or sub_plan = 'Individual' + WHERE sub_plan IS NULL ) , final AS ( @@ -18,4 +18,4 @@ WITH org_events AS ( GROUP BY 1 ) -SELECT * FROM final +SELECT * FROM final \ No newline at end of file From 9980b37d6382e22ec96f019690a86735fe9d28e1 Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Wed, 17 Jul 2024 12:03:04 -0700 Subject: [PATCH 15/17] Added top 10 to dim orgs model --- models/core/dim_orgs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 0b664ba..d743c05 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -33,7 +33,7 @@ WITH orgs AS ( ) -SELECT +SELECT TOP 10 org_id , created_at , num_users From e8373eb0ec40b600e762193d5bca66fccba41dfe Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Wed, 17 Jul 2024 12:53:54 -0700 Subject: [PATCH 16/17] Updated AWS_REGION from us-east-2 to us-west-2 in dbt_deploy.yml --- .github/workflows/dbt_deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index f2d78ac..fcc9175 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -31,7 +31,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-2 + AWS_REGION: us-west-2 - name: dbt build run: dbt build --target prod --select state:modified+ --defer --state ./ --exclude config.materialized:snapshot --full-refresh --profiles-dir ./ @@ -61,4 +61,4 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-2 + AWS_REGION: us-west-2 From e57cf4f51b1a2d57447eaefd58992e801cad9eef Mon Sep 17 00:00:00 2001 From: Insung Ko Date: Wed, 17 Jul 2024 13:00:03 -0700 Subject: [PATCH 17/17] removed top 10 in dim_orgs.sql and ran dbt build --- models/core/dim_orgs.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index d743c05..ae51d18 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -33,7 +33,7 @@ WITH orgs AS ( ) -SELECT TOP 10 +SELECT org_id , created_at , num_users @@ -42,4 +42,4 @@ SELECT TOP 10 , sub_price FROM orgs LEFT JOIN user_count USING (org_id) -LEFT JOIN subscriptions USING (org_id) \ No newline at end of file +LEFT JOIN subscriptions USING (org_id) \ No newline at end of file