From f4d634af8278ebcec23c821f0868d06c14a5d78c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 7 Nov 2022 17:01:39 -0600 Subject: [PATCH 1/9] add find and replace to json extract --- macros/pivot_json_extract.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/pivot_json_extract.sql b/macros/pivot_json_extract.sql index 025b683..e65edd3 100644 --- a/macros/pivot_json_extract.sql +++ b/macros/pivot_json_extract.sql @@ -2,7 +2,7 @@ {%- for property in list_of_properties -%} -replace( {{ fivetran_utils.json_extract(string, property) }}, '"', '') as {{ property | replace(' ', '_') | lower }} +replace( {{ fivetran_utils.json_extract(string, property) }}, '"', '') as {{ property | replace(' ', '_') | replace('.', '_') | lower }} {%- if not loop.last -%},{%- endif %} {% endfor -%} From 597143f6c65beb15b6d83135f4e3ac6f0fa61dca Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 21 Nov 2022 14:35:18 -0600 Subject: [PATCH 2/9] update macro --- macros/pivot_json_extract.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/macros/pivot_json_extract.sql b/macros/pivot_json_extract.sql index e65edd3..882c640 100644 --- a/macros/pivot_json_extract.sql +++ b/macros/pivot_json_extract.sql @@ -1,9 +1,13 @@ {% macro pivot_json_extract(string, list_of_properties) %} {%- for property in list_of_properties -%} +{%- if property is not string -%} +replace( {{ fivetran_utils.json_extract(string, property.name) }}, '"', '') as {{ property.alias if property.alias else property.name | replace(' ', '_') | replace('.', '_') | lower }} -replace( {{ fivetran_utils.json_extract(string, property) }}, '"', '') as {{ property | replace(' ', '_') | replace('.', '_') | lower }} +{%- else -%} +replace( {{ fivetran_utils.json_extract(string, property) }}, '"', '') as {{ property | replace(' ', '_') | lower }} +{%- endif -%} {%- if not loop.last -%},{%- endif %} {% endfor -%} From 8fe18e1eda137e0e0e1e7ca8213a0d64ec8bb99d Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 21 Nov 2022 14:44:32 -0600 Subject: [PATCH 3/9] update to accept more data types --- macros/pivot_json_extract.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/pivot_json_extract.sql b/macros/pivot_json_extract.sql index 882c640..e64b838 100644 --- a/macros/pivot_json_extract.sql +++ b/macros/pivot_json_extract.sql @@ -1,7 +1,7 @@ {% macro pivot_json_extract(string, list_of_properties) %} {%- for property in list_of_properties -%} -{%- if property is not string -%} +{%- if property is not mapping -%} replace( {{ fivetran_utils.json_extract(string, property.name) }}, '"', '') as {{ property.alias if property.alias else property.name | replace(' ', '_') | replace('.', '_') | lower }} {%- else -%} From 18f2e6e6ae97341da87ac34122c27ebb6408e7fa Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 21 Nov 2022 14:52:11 -0600 Subject: [PATCH 4/9] oops get rid of not --- macros/pivot_json_extract.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/pivot_json_extract.sql b/macros/pivot_json_extract.sql index e64b838..8891c2f 100644 --- a/macros/pivot_json_extract.sql +++ b/macros/pivot_json_extract.sql @@ -1,7 +1,7 @@ {% macro pivot_json_extract(string, list_of_properties) %} {%- for property in list_of_properties -%} -{%- if property is not mapping -%} +{%- if property is mapping -%} replace( {{ fivetran_utils.json_extract(string, property.name) }}, '"', '') as {{ property.alias if property.alias else property.name | replace(' ', '_') | replace('.', '_') | lower }} {%- else -%} From e2b211350ad4696bcb9312ef92272aebad5a0af2 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 21 Nov 2022 15:13:13 -0600 Subject: [PATCH 5/9] versioning --- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index c8e9b76..2380e93 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,4 @@ name: 'fivetran_utils' -version: '0.4.1' +version: '0.4.2' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6070c80..b0e10cd 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,4 +1,4 @@ name: 'fivetran_utils_integration_tests' -version: '0.4.1' +version: '0.4.2' config-version: 2 profile: 'integration_tests' \ No newline at end of file From c310ad87b007a99cf6fe3d40277f3e6cb2df5c77 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 21 Nov 2022 15:13:39 -0600 Subject: [PATCH 6/9] update pivot_json_extract desc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0dabc0f..b2900ed 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ This macro builds off of the `json_extract` macro in order to extract a list of ``` **Args:** * `string` (required): Name of the field which contains the json object. -* `list_of_properties` (required): List of the fields that you want to extract from the json object and pivot out into columns. Any spaces will be replaced by underscores in column names. +* `list_of_properties` (required): List of the fields that you want to extract from the json object and pivot out into columns. ---- ### string_agg ([source](macros/string_agg.sql)) From a2317feaf62760c19029b0bb21647596a5f6cf9c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 21 Nov 2022 15:13:46 -0600 Subject: [PATCH 7/9] add changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df06be..046827b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_fivetran_utils v0.4.2 + +## Updates +[PR #92](https://github.com/fivetran/dbt_fivetran_utils/pull/92) updates the `pivot_json_extract` macro to include additional formats of fields to be pivoted out into columns. Specifically, allowing for fields with `.` to be replaced with `_`, for metadata variables to accept dictionaries in addition to strings, and for aliasing of fields. + # dbt_fivetran_utils v0.4.1 ## Bug Fixes From 5f2d4e41a0382ed485488684469fd763a953a2c6 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 23 Nov 2022 10:53:38 -0600 Subject: [PATCH 8/9] fold to v0.4.1 --- CHANGELOG.md | 8 +++----- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 046827b..82dd9d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,11 @@ -# dbt_fivetran_utils v0.4.2 - -## Updates -[PR #92](https://github.com/fivetran/dbt_fivetran_utils/pull/92) updates the `pivot_json_extract` macro to include additional formats of fields to be pivoted out into columns. Specifically, allowing for fields with `.` to be replaced with `_`, for metadata variables to accept dictionaries in addition to strings, and for aliasing of fields. - # dbt_fivetran_utils v0.4.1 ## Bug Fixes [PR #91](https://github.com/fivetran/dbt_fivetran_utils/pull/91) updates dispatch from `{{ dbt_utils. }}` to `{{ dbt. }}` for additional [cross-db macros](https://docs.google.com/spreadsheets/d/1xF_YwJ4adsnjFkUbUm8-EnEL1r_C-9_OI_pP4m4FlJU/edit#gid=1062533692) missed in the `fivetran_utils.union_relations()` macro. +## Updates +[PR #92](https://github.com/fivetran/dbt_fivetran_utils/pull/92) updates the `pivot_json_extract` macro to include additional formats of fields to be pivoted out into columns. Specifically, allowing for fields with `.` to be replaced with `_`, for metadata variables to accept dictionaries in addition to strings, and for aliasing of fields. + # dbt_fivetran_utils v0.4.0 ## Bug Fixes diff --git a/dbt_project.yml b/dbt_project.yml index 2380e93..c8e9b76 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,4 @@ name: 'fivetran_utils' -version: '0.4.2' +version: '0.4.1' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index b0e10cd..6070c80 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,4 +1,4 @@ name: 'fivetran_utils_integration_tests' -version: '0.4.2' +version: '0.4.1' config-version: 2 profile: 'integration_tests' \ No newline at end of file From 6ecb0ab7bb2b2cfebf2fb366c699bc950c55c60a Mon Sep 17 00:00:00 2001 From: Renee Li <91097070+fivetran-reneeli@users.noreply.github.com> Date: Tue, 29 Nov 2022 11:53:18 -0600 Subject: [PATCH 9/9] Update CHANGELOG.md Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82dd9d0..d49a9f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,10 @@ # dbt_fivetran_utils v0.4.1 ## Bug Fixes -[PR #91](https://github.com/fivetran/dbt_fivetran_utils/pull/91) updates dispatch from `{{ dbt_utils. }}` to `{{ dbt. }}` for additional [cross-db macros](https://docs.google.com/spreadsheets/d/1xF_YwJ4adsnjFkUbUm8-EnEL1r_C-9_OI_pP4m4FlJU/edit#gid=1062533692) missed in the `fivetran_utils.union_relations()` macro. +- Updates dispatch from `{{ dbt_utils. }}` to `{{ dbt. }}` for additional [cross-db macros](https://docs.google.com/spreadsheets/d/1xF_YwJ4adsnjFkUbUm8-EnEL1r_C-9_OI_pP4m4FlJU/edit#gid=1062533692) missed in the `fivetran_utils.union_relations()` macro. ([PR #91](https://github.com/fivetran/dbt_fivetran_utils/pull/91)) ## Updates -[PR #92](https://github.com/fivetran/dbt_fivetran_utils/pull/92) updates the `pivot_json_extract` macro to include additional formats of fields to be pivoted out into columns. Specifically, allowing for fields with `.` to be replaced with `_`, for metadata variables to accept dictionaries in addition to strings, and for aliasing of fields. +- Updates the `pivot_json_extract` macro to include additional formats of fields using a new `name` and `alias` argument to be pivoted out into columns. Specifically, allowing for fields with `.` to be replaced with `_`, for metadata variables to accept dictionaries in addition to strings, and for aliasing of fields. ([PR #92](https://github.com/fivetran/dbt_fivetran_utils/pull/92)) # dbt_fivetran_utils v0.4.0