Release/v0.14.0

CHANGELOG.md

@@ -1,3 +1,24 @@
+# dbt_hubspot_source v0.14.0
+[PR #122](https://github.com/fivetran/dbt_hubspot_source/pull/122) includes the following updates:
+
+## 🚨 Breaking Change 🚨 
+- When including a passthrough `property_hs_*` column, you now have the option to include the corresponding, human-readable label in the staging models. 
+  - See the [Adding property label section](https://github.com/fivetran/dbt_hubspot_source#adding-property-label) of the README for instructions on how to enable this feature! 
+  - This update applies to models:
+    - `stg_hubspot__company`
+    - `stg_hubspot__contact`
+    - `stg_hubspot__deal`
+    - `stg_hubspot__ticket`
+
+## Features
+- Added the following staging models, along with documentation and tests:
+  - `stg_hubspot__property`
+  - `stg_hubspot__property_option`
+
+## Bug fixes
+- Updated macro `remove_duplicate_and_prefix_from_columns` to accommodate incoming custom column names containing characters such as `-` or `$` that are not permitted. The resulting column name will have these characters removed or replaced in its `stg_*` model. 
+- Removed extra comma from `stg_hubspot__ticket`, which was causing compilation issues when passing through all columns. 
+
 # dbt_hubspot_source v0.13.0
 
 ## 🚨 Breaking Changes 🚨

README.md

@@ -135,7 +135,7 @@ vars:
   hubspot__ticket_pass_through_columns:
     - name:           "property_mmm"
       alias:          "mmm"
-    - name:           "property_bop"
+    - name:           "property_hs_bop"
       alias:          "bop"
 ```
 **Alternatively**, if you would like to simply pass through **all columns** in the above four tables, add the following configuration to your dbt_project.yml. Note that this will override any `hubspot__[table_name]_pass_through_columns` variables.
@@ -144,6 +144,28 @@ vars:
 vars:
   hubspot__pass_through_all_columns: true # default is false
 ```
+
+### Adding property label
+For `property_hs_*` columns, you can enable the corresponding, human-readable `property_option`.`label` to be included in the staging models. 
+- **Note** you cannot bring in labels if using `hubspot__pass_through_all_columns: true`.` 
+- To enable labels for a given property, set the property attribute `use_property_label: true`, using the below format.
+
+```yml
+vars:
+  hubspot__ticket_pass_through_columns:
+    - name: "property_hs_fieldname"
+      alias: "fieldname"
+      use_property_label: true
+```
+  Alternatively, you can enable labels for all passthrough properties by using variable `hubspot__enable_all_property_labels: true`, formatted like the below example. 
+
+```yml
+vars:
+  hubspot__enable_all_property_labels: true
+  hubspot__ticket_pass_through_columns:
+    - name: "property_hs_fieldname1"
+    - name: "property_hs_fieldname2"
+```
 ### Including calculated fields
 This package also provides the ability to pass calculated fields through to the `company`, `contact`, `deal`, and `ticket` staging models. If you would like to add a calculated field to any of the mentioned staging models, you may configure the respective `hubspot__[table_name]_calculated_fields` variables with the `name` of the field you would like to create, and the `transform_sql` which will be the actual calculation that will make up the calculated field.
 ```yml

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.13.0'
+version: '0.14.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -47,6 +47,8 @@ vars:
     engagement_note: "{{ source('hubspot','engagement_note') }}"
     engagement_task: "{{ source('hubspot','engagement_task') }}"
     owner: "{{ source('hubspot','owner') }}"
+    property: "{{ source('hubspot','property') }}"
+    property_option: "{{ source('hubspot','property_option') }}"
     ticket_company: "{{ source('hubspot','ticket_company') }}"
     ticket_contact: "{{ source('hubspot','ticket_contact') }}"
     ticket_deal: "{{ source('hubspot','ticket_deal') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_source_integration_tests_7
+      schema: hubspot_source_integration_tests_99
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_source_integration_tests_7
+      schema: hubspot_source_integration_tests_99
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_source_integration_tests_7
+      schema: hubspot_source_integration_tests_99
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_source_integration_tests_7
+      schema: hubspot_source_integration_tests_99
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_source_integration_tests_7
+      schema: hubspot_source_integration_tests_99
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,14 +1,18 @@
 name: 'hubspot_source_integration_tests'
-version: '0.13.0'
+version: '0.14.0'
 profile: 'integration_tests'
 config-version: 2
 models:
   hubspot_source:
     +schema:
 vars:
-  hubspot_schema: hubspot_source_integration_tests_7
+  hubspot_schema: hubspot_source_integration_tests_99
   hubspot_source:
     hubspot_service_enabled: true
+    # hubspot_sales_enabled: true # enable when generating docs
+    # hubspot_marketing_enabled: true # enable when generating docs
+    # hubspot_contact_merge_audit_enabled: true # enable when generating docs
+    # hubspot_using_all_email_events: true # enable when generating docs
     hubspot_company_property_history_identifier: "company_property_history_data"
     hubspot_company_identifier: "company_data"
     hubspot_contact_identifier: "contact_data"
@@ -43,6 +47,8 @@ vars:
     hubspot_engagement_note_identifier: "engagement_note_data"
     hubspot_engagement_task_identifier: "engagement_task_data"
     hubspot_owner_identifier: "owner_data"
+    hubspot_property_identifier: "property_data"
+    hubspot_property_option_identifier: "property_option_data"
     hubspot_ticket_company_identifier: "ticket_company_data"
     hubspot_ticket_contact_identifier: "ticket_contact_data"
     hubspot_ticket_deal_identifier: "ticket_deal_data"

integration_tests/requirements.txt

@@ -4,5 +4,4 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
-oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437
+dbt-databricks>=1.3.0,<2.0.0

integration_tests/seeds/property_data.csv

@@ -0,0 +1,11 @@
+_fivetran_id,_fivetran_synced,calculated,created_at,description,field_type,group_name,hubspot_defined,hubspot_object,label,name,show_currency_symbol,type,updated_at
++mlGrLvmij8zluyeRByh7DPbXpM=,2023-11-11 08:18:16.519000 UTC,false,2023-05-03 08:51:08.199000 UTC,M0xKTEL9t51+vD5ztRfm+A==,date,fVQYOcf70/KsHU/KFoXQfQ==,,contact,test_pro,test_pro,false,date,2023-05-03 08:51:08.199000 UTC
+6fnvV+pkZR3di3USRclW2NZMNTA=,2023-11-11 08:18:15.151000 UTC,false,2019-08-06 02:41:52.364000 UTC,k+VMRhjE/W4QB470Jor1yA==,date,j8mgV9/MYzXvxQKD3c0AHw==,true,deal,Close Date,closedate,,datetime,2023-06-20 22:32:28.368000 UTC
+5yTEFwFlLPTKlzXc3PviX5DMdJ4=,2023-11-11 08:18:16.569000 UTC,false,2019-08-06 02:41:09.071000 UTC,nRGOJjIZrs4d0WJLkC15fg==,date,fVQYOcf70/KsHU/KFoXQfQ==,true,contact,Close Date,closedate,,datetime,2022-05-27 20:35:26.640000 UTC
+u0bk1tHdJJir2SmRkwo1r22t2PM=,2023-11-11 08:18:15.837000 UTC,false,2019-08-06 02:41:44.665000 UTC,HRIf2XFKVmDQxMzI2w3mcA==,date,XArNQdwSO+DY7119vj+JEw==,true,company,Close Date,closedate,,datetime,2022-05-27 22:49:16.893000 UTC
+RTJ5KTwQe/zMYl8uKwNFlOeNWUY=,2023-11-11 08:18:15.792000 UTC,false,2022-11-30 10:52:48.277000 UTC,1B2M2Y8AsgTpgAmY7PhCfg==,date,XArNQdwSO+DY7119vj+JEw==,,company,test_date,test_date,false,date,2022-11-30 10:52:48.277000 UTC
+sv5Aa2A1+wJwoKRX4Zfe3CTEHEE=,2023-11-11 08:18:16.575000 UTC,false,2019-08-06 02:41:09.148000 UTC,jjSyQlM9DMhpVol12tp4fg==,date,fVQYOcf70/KsHU/KFoXQfQ==,true,contact,Create Date,createdate,,datetime,2023-05-01 20:02:33.973000 UTC
+GjTSotYsbt+HoRY5e6bHRgNJYFI=,2023-11-11 08:18:36.368000 UTC,false,2019-08-06 02:42:48.261000 UTC,nBljQWslMNrEDwXeYmRYyg==,date,0NoL94xOqMRlkFF+GAX6QQ==,true,product,Create Date,createdate,,datetime,2022-05-28 01:26:21.260000 UTC
+6iFNLSeY0Fz5DgwskGjrMoW3JIg=,2023-11-11 08:18:13.861000 UTC,false,2019-08-06 02:42:38.781000 UTC,lMRanmvFHjNC0pQdw79H5Q==,date,kdJdJeqzIqBPlKTSRZoyGQ==,true,line_item,Create Date,createdate,,datetime,2022-05-28 01:31:47.067000 UTC
+onbRYvHwxXS3NwSP2zaoD+Mv/us=,2023-11-11 08:18:36.567000 UTC,false,2019-08-06 02:43:00.503000 UTC,MD3INB3ab4k2LqaTom0eEA==,date,CfZkcOOMcc/dg5a9ofwkUg==,true,ticket,Create date,createdate,,datetime,2022-08-18 12:55:35.731000 UTC
+XhKQ/eq5pqJEB6smEKCPSSyfUH4=,2023-11-11 08:18:15.836000 UTC,false,2019-08-06 02:41:44.631000 UTC,wyvwVnWdqsdNRapzG8Y/RA==,date,XArNQdwSO+DY7119vj+JEw==,true,company,Create Date,createdate,,datetime,2023-05-01 19:33:57.873000 UTC

integration_tests/seeds/property_option_data.csv

@@ -0,0 +1,21 @@
+label,property_id,_fivetran_synced,display_order,hidden,value
+psScnliy99RZj01H2CO0Bg==,QbL3/uGeJqHy1PwdKiAryA==,2023-07-06 05:18:21.205000 UTC,0,false,Rk7Ftpi8GPQtaS7MlroMGQ==
+pSz+WBU1TmUa4LuUa1rgkQ==,QbL3/uGeJqHy1PwdKiAryA==,2023-07-06 05:18:21.205000 UTC,1,false,LA3JfM9iZ4SdcrsahteEeA==
+AvzvMEDzIxXmOQHpmJkR2Q==,QbL3/uGeJqHy1PwdKiAryA==,2023-07-06 05:18:21.205000 UTC,2,false,T9S3rtCg0J+FcavCHGDYtw==
+82p3OmCpiuf6fXiV9CNUGg==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,54,false,MpeVBohL7jkWOlWD1n2MEQ==
+iXyRZw2q18uicoYZGNbgag==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,69,false,ag2ORssJEO3Ix9VleEtdQg==
+kNCldBZipThyNGTWrrfg2A==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,61,false,DCbZyLGAXmlPv5n4SfYx9Q==
+nXBeZz/aTEAYfzniBd/75g==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,53,false,isnpiTWMqdJGp0MdQexQlA==
+Y6UJ4WGHacm+BqSY+9LIWw==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,68,false,NxElfDRGA3UVvLxOm2zBuA==
+9/yByDltisVJ/ItQKfpDug==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,58,false,vIGBU9BcEMJOtKde0VQiqw==
+olS9+/n99uTy3AIPobhBFQ==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,73,false,0TcZBF4uKrmsLk3qXWb/Hg==
+B1Y77Uvfwx7eFghmY0fMnA==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,64,false,/cjAina18SVT3IObiSTw3w==
+K9C6Ej/GCH0SeJeYNHAilQ==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,55,false,1m8HK+BNhT7R+gsuVesdxw==
+pywmmY+CL/LxU1GOsa6inw==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,63,false,kqfrguMbRr20bZ8iN/xUww==
+0ZCS71MB2VSJEfh6gX+MRA==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,59,false,/6WH59+ebbdk/66q3ur5Cw==
+o9hmABcYPHTCe5qrVu1lIQ==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,75,false,JUbNYa3Fg/uz4J33ml2QoQ==
+du4+7EMJLifwRGQjGBJNbA==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,62,false,oGDrsJou/ln6KJivl+ruWQ==
+pTcKx6PCI9FNH+BrqtoqDg==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,60,false,+rN2xb29RGurpWYBfnHC5g==
+OHV/B1A6v1fd9MipdPIUKQ==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,70,false,jDwQNnk3/oWaB7pqsOXBLA==
+rrQoVUoKzGzFA6Lk2mrmGg==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,76,false,IrC/vHIdyaGW7KTyFPL+LA==
+6v8b3yT8/+DhTimhv/UaEg==,stMBaoy9Irlfo81ajZF/Ww==,2023-07-06 05:18:20.949000 UTC,65,false,Z/qUdomTCUc+d/5MgyA10g==

macros/add_property_labels.sql

@@ -0,0 +1,42 @@
+{% macro add_property_labels(passthrough_var_name, cte_name) %}
+
+select {{ cte_name }}.*
+
+{%- if var(passthrough_var_name, []) != [] -%}
+  {%- set col_list = var(passthrough_var_name) -%}
+
+  {%- for col in col_list -%} -- Create label cols
+    {%- if col.add_property_label or var('hubspot__enable_all_property_labels', false) -%}
+      {%- set col_alias = (col.alias | default(col.name)) %}
+      , {{ col.name }}_option.property_option_label as {{ col_alias }}_label
+    {% endif -%}
+  {%- endfor %}
+
+  from {{ cte_name }}
+
+  {% for col in col_list -%} -- Create joins
+    {%- if col.add_property_label or var('hubspot__enable_all_property_labels', false) -%}
+      {%- set col_alias = (col.alias | default(col.name)) %}
+
+  left join -- create subset of property and property_options for property in question
+    (select 
+      property_option.property_option_value, 
+      property_option.property_option_label
+    from {{ ref('stg_hubspot__property_option') }} as property_option
+    join {{ ref('stg_hubspot__property') }} as property
+      on property_option.property_id = property._fivetran_id
+      and property_option.property_option_label = property.property_label
+    where property.property_name = '{{ col.name.replace('property_', '') }}'
+    ) as {{ col.name }}_option
+
+    on cast({{ cte_name }}.{{ col_alias }} as {{ dbt.type_string() }})
+      = cast({{ col.name }}_option.property_option_value as {{ dbt.type_string() }})
+
+    {% endif -%}
+  {%- endfor %}
+
+{%- else -%}
+  from {{ cte_name }}
+
+{%- endif -%}
+{% endmacro %}

macros/get_property_columns.sql

@@ -0,0 +1,22 @@
+{% macro get_property_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_id", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "calculated", "datatype": dbt.type_boolean()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "description", "datatype": dbt.type_string()},
+    {"name": "field_type", "datatype": dbt.type_string()},
+    {"name": "group_name", "datatype": dbt.type_string()},
+    {"name": "hubspot_defined", "datatype": dbt.type_boolean()},
+    {"name": "hubspot_object", "datatype": dbt.type_string()},
+    {"name": "label", "datatype": dbt.type_string()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "show_currency_symbol", "datatype": dbt.type_boolean()},
+    {"name": "type", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_property_option_columns.sql

@@ -0,0 +1,14 @@
+{% macro get_property_option_columns() %}
+
+{% set columns = [
+    {"name": "label", "datatype": dbt.type_string()},
+    {"name": "property_id", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "display_order", "datatype": dbt.type_int()},
+    {"name": "hidden", "datatype": dbt.type_boolean()},
+    {"name": "value", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/remove_duplicate_and_prefix_from_columns.sql

@@ -6,17 +6,17 @@
         {%- for dupe in columns if col.name[prefix|length:]|lower == dupe.name|lower -%}
         {%- do duplicate_exclude.append(col.name) -%}
         {%- do duplicate_exclude.append(dupe.name) -%}
-        , {{ col.name }} as {{ col.name[prefix|length:] }}
+        , {{ adapter.quote(col.name) }} as {{ dbt_utils.slugify(col.name[prefix|length:]) }}
         {%- endfor %}
         {% endfor %}
 
         {# Loop again to find non-duplicates #}
         {% for col in columns if col.name not in exclude %}
         {%- if col.name|lower not in duplicate_exclude|lower -%}
         {% if col.name[:prefix|length]|lower == prefix %}
-        , {{ col.name }} as {{ col.name[prefix|length:] }}
+        , {{ adapter.quote(col.name) }} as {{ dbt_utils.slugify(col.name[prefix|length:]) }}
         {%- else %}
-        , {{ col.name }}
+        , {{ adapter.quote(col.name) }}
         {%- endif -%}
         {%- endif -%}
         {% endfor %}

models/src_hubspot.yml

@@ -1093,7 +1093,60 @@ sources:
           - name: type
             description: The type of owner.
           - name: updated_at
-            description: A timestamp for when the owner was last updated.
+
+      - name: property
+        identifier: "{{ var('hubspot_property_identifier', 'property')}}"
+        description: Each record represents a property.
+        config:
+          enabled: "{{ var('hubspot_sales_enabled', true) }}"
+        columns:
+          - name: _fivetran_id
+            description: Fivetran generated id. Joins to `property_id` in the `property_option` table.
+          - name: _fivetran_synced
+            description: '{{ doc("_fivetran_synced") }}'
+          - name: calculated
+            description: Indicates if the property is calculated by a HubSpot process
+          - name: created_at
+            description: Timestamp representing when the property was created.
+          - name: description
+            description: A description of the property. 
+          - name: field_type
+            description: One of textarea, text, date, file, number, select, radio, checkbox, or booleancheckbox.
+          - name: group_name
+            description: The name of the property group that the property belongs to.
+          - name: hubspot_defined
+            description: This will be true for default properties that are built into HubSpot.
+          - name: hubspot_object
+            description: If this property is related to other objects, the object will be listed here.
+          - name: label
+            description: A human readable label for the property.
+          - name: name
+            description: The internal name of the property.
+          - name: show_currency_symbol
+            description: If true, the property will display the currency symbol set in the portal settings.
+          - name: type
+            description: One of string, number, date, datetime, or enumeration.
+          - name: updated_at
+            description: Timestamp representing when the property was last updated.
+
+      - name: property_option
+        identifier: "{{ var('hubspot_property_option_identifier', 'property_option')}}"
+        description: Each record represents an option for the related property.
+        config:
+          enabled: "{{ var('hubspot_sales_enabled', true) }}"
+        columns:
+          - name: _fivetran_synced
+            description: '{{ doc("_fivetran_synced") }}'
+          - name: label
+            description: The label of the option displayed inside the HubSpot app.
+          - name: property_id
+            description: The ID of the related property.
+          - name: display_order
+            description: Order of options displayed in Hubspot, starting with the lowest positive integer value. Values of -1 will cause the option to be displayed after any positive values.
+          - name: hidden
+            description: Boolean if the option will be displayed in HubSpot
+          - name: value
+            description: The internal value of the option.
 
       - name: ticket_company
         identifier: "{{ var('hubspot_ticket_company_identifier', 'ticket_company')}}"

models/stg_hubspot__company.sql

@@ -63,7 +63,10 @@ with base as (
     from macro
 
 {% endif %}
+
+), joined as (
+    {{ add_property_labels('hubspot__company_pass_through_columns', cte_name='fields') }}
 )
 
 select *
-from fields
+from joined

models/stg_hubspot__contact.sql

@@ -60,8 +60,10 @@ with base as (
 
     from macro
 {% endif %}    
-
+
+), joined as (
+    {{ add_property_labels('hubspot__contact_pass_through_columns', cte_name='fields') }}
 )
 
 select *
-from fields
+from joined