Add service end models

.buildkite/scripts/run_models.sh

@@ -20,7 +20,12 @@ dbt seed --target "$db" --full-refresh
 dbt compile --target "$db" --select hubspot # source does not compile at this time
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh
+dbt run --target "$db" --vars '{hubspot_service_enabled: true}' --full-refresh
+dbt run --target "$db" --vars '{hubspot_service_enabled: true}'
+dbt test --target "$db"
+dbt run --vars '{hubspot_service_enabled: true, hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh
+dbt run --vars '{hubspot_service_enabled: true, hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db"
+dbt test --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh
 dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false}' --target "$db" --full-refresh
 dbt test --target "$db"

CHANGELOG.md

@@ -1,3 +1,31 @@
+# dbt_hubspot v0.14.0
+
+## 🚨 Breaking Changes 🚨
+- Within the source package the `created_at` and `closed_at` fields in the below mentioned staging models have been renamed to `created_date` and `closed_date` respectively to be consistent with the source data. Additionally, this will ensure there are no duplicate column errors when passing through all `property_*` columns, which could potentially conflict with `property_created_at` or `property_closed_at`. ([PR #119](https://github.com/fivetran/dbt_hubspot_source/pull/119))
+  - `stg_hubspot__company`
+    - Impacts `hubspot__companies`
+  - `stg_hubspot__contact`
+    - Impacts `hubspot__contacts`
+  - `stg_hubspot__deal`
+    - Impacts `hubspot__deals`
+  - `stg_hubspot__ticket`
+    - Impacts `hubspot__tickets`
+
+## New Model Alert 😮
+Introducing Service end models! These are disabled by default but can be enabled by setting `hubspot_service_enabled` to `true` ([PR #123](https://github.com/fivetran/dbt_hubspot/pull/123)):
+  - `hubspot__tickets` - [Docs](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__tickets)
+  - `hubspot__daily_ticket_history` - [Docs](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__daily_ticket_history)
+    - See additional configurations for the history model in [README](https://github.com/fivetran/dbt_hubspot/tree/main#daily-ticket-history)
+
+## Features
+- Addition of the following variables to allow the disabling of the `*_property_history` models if they are not being leveraged. All variables are `true` by default. ([PR #122](https://github.com/fivetran/dbt_hubspot/pull/122))
+  - `hubspot_company_property_history_enabled`
+  - `hubspot_contact_property_history_enabled`
+  - `hubspot_deal_property_history_enabled`
+
+## Under the Hood
+- Updates to the seed files and seed file configurations for the package integration tests to ensure updates are properly tested. ([PR #122](https://github.com/fivetran/dbt_hubspot/pull/122))
+
 # dbt_hubspot v0.13.0
 ## 🚨 Breaking Changes 🚨
 - This release will be a breaking change due to the removal of below dependencies.

README.md

@@ -35,6 +35,8 @@ The following table provides a detailed list of all models materialized within t
 | [hubspot__deals](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__deals)            | Each record represents a deal in Hubspot, enriched with metrics about engagement activities.                         |
 | [hubspot__deal_stages](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__deal_stages)            | Each record represents a deal stage in Hubspot, enriched with metrics deal activities.                         |
 | [hubspot__deal_history](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__deal_history)    | Each record represents a change to a deal in Hubspot, with `valid_to` and `valid_from` information.                  |
+| [hubspot__tickets](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__tickets)    | Each record represents a ticket in Hubspot, enriched with metrics about engagement activities and information on associated deals, contacts, companies, and owners.                  |
+| [hubspot__daily_ticket_history](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__daily_ticket_history)    | Each record represents a ticket's day in Hubspot with tracked properties pivoted out into columns.               |
 | [hubspot__email_campaigns](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__email_campaigns) | Each record represents a email campaign in Hubspot, enriched with metrics about email activities.                    |
 | [hubspot__email_event_*](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__email_event_bounce)   | Each record represents an email event in Hubspot, joined with relevant tables to make them analysis-ready.           |
 | [hubspot__email_sends](https://fivetran.github.io/dbt_hubspot/#!/model/model.hubspot.hubspot__email_sends)     | Each record represents a sent email in Hubspot, enriched with metrics about opens, clicks, and other email activity. |
@@ -57,13 +59,21 @@ dispatch:
     search_order: ['spark_utils', 'dbt_utils']
 ```
 
+### Database Incremental Strategies 
+Some of the models (`+hubspot__daily_ticket_history`) in this package are materialized incrementally. We have chosen `insert_overwrite` as the default strategy for **BigQuery** and **Databricks** databases, as it is only available for these dbt adapters. For **Snowflake**, **Redshift**, and **Postgres** databases, we have chosen `delete+insert` as the default strategy.
+
+`insert_overwrite` is our preferred incremental strategy because it will be able to properly handle updates to records that exist outside the immediate incremental window. That is, because it leverages partitions, `insert_overwrite` will appropriately update existing rows that have been changed upstream instead of inserting duplicates of them--all without requiring a full table scan.
+
+`delete+insert` is our second-choice as it resembles `insert_overwrite` but lacks partitions. This strategy works most of the time and appropriately handles incremental loads that do not contain changes to past records. However, if a past record has been updated and is outside of the incremental window, `delete+insert` will insert a duplicate record. 😱
+> Because of this, we highly recommend that **Snowflake**, **Redshift**, and **Postgres** users periodically run a `--full-refresh` to ensure a high level of data quality and remove any possible duplicates.
+
 ## Step 2: Install the package
 Include the following hubspot package version in your `packages.yml` file:
 > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 ```yaml
 packages:
   - package: fivetran/hubspot
-    version: [">=0.13.0", "<0.14.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.14.0", "<0.15.0"] # we recommend using ranges to capture non-breaking changes automatically
 
 ```
 Do **NOT** include the `hubspot_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -97,6 +107,7 @@ vars:
   hubspot_contact_list_enabled: false                     # Disables contact list models
   hubspot_contact_list_member_enabled: false              # Disables contact list member models
   hubspot_contact_property_enabled: false                 # Disables the contact property models
+  hubspot_contact_property_history_enabled: false         # Disables the contact property history models
   hubspot_email_event_enabled: false                      # Disables all email_event models and functionality
   hubspot_email_event_bounce_enabled: false
   hubspot_email_event_click_enabled: false
@@ -118,9 +129,11 @@ vars:
 
   hubspot_sales_enabled: false                            # Disables all sales models
   hubspot_company_enabled: false
+  hubspot_company_property_history_enabled: false         # Disables the company property history models
   hubspot_deal_enabled: false
   hubspot_deal_company_enabled: false
   hubspot_deal_contact_enabled: false
+  hubspot_deal_property_history_enabled: false            # Disables the deal property history models
   hubspot_engagement_enabled: false                       # Disables all engagement models and functionality
   hubspot_engagement_contact_enabled: false
   hubspot_engagement_company_enabled: false
@@ -133,11 +146,10 @@ vars:
   hubspot_owner_enabled: false
 
   # Service
-  hubspot_service_enabled: true                           # Enables all service models
-  hubspot_ticket_deal_enabled: true
+  hubspot_service_enabled: true                           # Enables all service/ticket models. Default = false
+  hubspot_ticket_deal_enabled: true                       # Default = false
 ```
 ## (Optional) Step 5: Additional configurations
-<details><summary>Expand for configurations</summary>
 
 ### Configure email metrics
 This package allows you to specify which email metrics (total count and total unique count) you would like to be calculated for specified fields within the `hubspot__email_campaigns` model. By default, the `email_metrics` variable below includes all the shown fields. If you would like to remove any field metrics from the final model, you may copy and paste the below snippet within your root `dbt_project.yml` and remove any fields you want to be ignored in the final model.
@@ -210,6 +222,35 @@ vars:
   hubspot_using_all_email_events: false # True by default
 ```
 
+### Daily ticket history
+The `hubspot__daily_ticket_history` model is disabled by default, but will materialize if `hubspot_service_enabled` is set to `true`. See additional configurations for this model below.
+
+> **Note**: `hubspot__daily_ticket_history` and its parent intermediate models are incremental. After making any of the below configurations, you will need to run a full refresh.
+
+#### **Tracking ticket properties**
+By default, `hubspot__daily_ticket_history` will track each ticket's state, pipeline, and pipeline stage and pivot these properties into columns. However, any property from the source `TICKET_PROPERTY_HISTORY` table can be tracked and pivoted out into columns. To add other properties to this end model, add the following configuration to your `dbt_project.yml` file:
+
+```yml
+vars:
+  hubspot__ticket_property_history_columns:
+    - the
+    - list
+    - of 
+    - property
+    - names
+```
+
+#### **Extending ticket history past closing date**
+This package will create a row in `hubspot__daily_ticket_history` for each day that a ticket is open, starting at its creation date. A Hubspot ticket can be altered after being closed, so its properties can change after this date.
+
+By default, the package will track a ticket up to its closing date (or the current date if still open). To capture post-closure changes, you may want to extend a ticket's history past the close date. To do so, add the following configuration to your root dbt_project.yml file:
+
+```yml
+vars:
+  hubspot:
+    ticket_history_extension_days: integer_number_of_days # default = 0
+```
+
 ### Changing the Build Schema
 By default this package will build the HubSpot staging models within a schema titled (<target_schema> + `_stg_hubspot`) and HubSpot final models within a schema titled (<target_schema> + `hubspot`) in your target database. If this is not where you would like your modeled HubSpot data to be written to, add the following configuration to your root `dbt_project.yml` file:
 
@@ -230,7 +271,6 @@ If an individual source table has a different name than the package expects, add
 vars:
     hubspot_<default_source_table_name>_identifier: your_table_name
 ```
-</details>
 
 ## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™
 <details><summary>Expand for details</summary>
@@ -246,7 +286,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/hubspot_source
-      version: [">=0.12.0", "<0.13.0"]
+      version: [">=0.13.0", "<0.14.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.13.0'
+version: '0.14.0'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
@@ -42,6 +42,16 @@ vars:
     engagement_meeting: "{{ ref('stg_hubspot__engagement_meeting') }}"
     engagement_note: "{{ ref('stg_hubspot__engagement_note') }}"
     engagement_task: "{{ ref('stg_hubspot__engagement_task') }}"
+
+    ticket_company: "{{ ref('stg_hubspot__ticket_company') }}"
+    ticket_contact: "{{ ref('stg_hubspot__ticket_contact') }}"
+    ticket_deal: "{{ ref('stg_hubspot__ticket_deal') }}"
+    ticket_engagement: "{{ ref('stg_hubspot__ticket_engagement') }}"
+    ticket_pipeline: "{{ ref('stg_hubspot__ticket_pipeline') }}"
+    ticket_pipeline_stage: "{{ ref('stg_hubspot__ticket_pipeline_stage') }}"
+    ticket_property_history: "{{ ref('stg_hubspot__ticket_property_history') }}"
+    ticket: "{{ ref('stg_hubspot__ticket') }}"
+
     hubspot_contact_merge_audit_enabled: false
 models:
   hubspot:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_integration_tests'
-version: '0.13.0'
+version: '0.14.0'
 
 profile: 'integration_tests'
 config-version: 2
@@ -49,9 +49,10 @@ vars:
     hubspot_ticket_pipeline_stage_identifier: "ticket_pipeline_stage_data"
     hubspot_ticket_property_history_identifier: "ticket_property_history_data"
     hubspot_ticket_identifier: "ticket_data"
-    contact_list: "{{ fivetran_utils.seed_data_helper('contact_list_data', ['postgres']) }}"
-    email_event_sent: "{{ fivetran_utils.seed_data_helper('email_event_sent_data', ['snowflake','postgres']) }}"
-    email_event_dropped: "{{ fivetran_utils.seed_data_helper('email_event_dropped_data', ['snowflake','postgres']) }}"
+    hubspot_contact_list_identifier: "contact_list_data"
+    hubspot_email_event_sent_identifier: "email_event_sent_data"
+    hubspot_email_event_dropped_identifier: "email_event_dropped_data"
+
 seeds:
   hubspot_integration_tests:
     +quote_columns: "{{ true if target.type == 'redshift' else false }}"
@@ -136,20 +137,25 @@ seeds:
       +column_types:
         stage_id: "{{ 'varchar(100)'  if target.type in ('redshift','postgres') else 'string'}}"
     email_event_sent_data_snowflake:
+      +alias: email_event_sent_data
       +enabled: "{{ true if target.type == 'snowflake' else false }}"
     email_event_sent_data_postgres:
+      +alias: email_event_sent_data
       +enabled: "{{ true if target.type == 'postgres' else false }}"
     email_event_sent_data:
       +enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}"
     email_event_dropped_data_snowflake:
+      +alias: email_event_dropped_data
       +enabled: "{{ true if target.type == 'snowflake' else false }}"
     email_event_dropped_data_postgres:
+      +alias: email_event_dropped_data
       +enabled: "{{ true if target.type == 'postgres' else false }}"
     email_event_dropped_data:
       +enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}"
     contact_list_data:
       +enabled: "{{ true if target.type != 'postgres' else false }}"
     contact_list_data_postgres:
+      +alias: contact_list_data
       +enabled: "{{ true if target.type == 'postgres' else false }}"
     email_event_data: 
       +column_types:
@@ -158,7 +164,10 @@ seeds:
         created: timestamp
         obsoleted_by_created: timestamp
         sent_by_created: timestamp
+    ticket_property_history_data:
+      +column_types:
+        timestamp_instant: timestamp
 
 dispatch:
   - macro_namespace: dbt_utils
-    search_order: ['spark_utils', 'dbt_utils']
+    search_order: ['spark_utils', 'dbt_utils']

integration_tests/requirements.txt

@@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437