Source Hubspot: refactor PropertyHistory stream (#14666)

* #291 oncall. Source Hubspot: disable Incremental mode for PropertyHistory stream and decrease number of http requests made * #291 oncall: upd changelog * #291 oncall: rm property_history stream from full_refresh SAT * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jul 14, 2022 · 400f4e6 · 400f4e6
1 parent 89e78a6
commit 400f4e6
Show file tree

Hide file tree

Showing 7 changed files with 11 additions and 23 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -406,7 +406,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.72
+  dockerImageTag: 0.1.73
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3693,7 +3693,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.1.72"
+- dockerImage: "airbyte/source-hubspot:0.1.73"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.72
+LABEL io.airbyte.version=0.1.73
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml
@@ -68,8 +68,6 @@ tests:
         "tickets": [ "properties", "hs_time_in_2" ]
         "tickets": [ "properties", "hs_time_in_3" ]
         "tickets": [ "properties", "hs_time_in_4" ]
-        "property_history": [ "property", "hs_time_in_lead" ]
-        "property_history": [ "property", "hs_time_in_subscriber" ]
     - config_path: "secrets/config_oauth.json"
       configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"
       ignored_fields:
@@ -103,5 +101,3 @@ tests:
         "tickets": [ "properties", "hs_time_in_2" ]
         "tickets": [ "properties", "hs_time_in_3" ]
         "tickets": [ "properties", "hs_time_in_4" ]
-        "property_history": [ "property", "hs_time_in_lead" ]
-        "property_history": [ "property", "hs_time_in_subscriber" ]
diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json
@@ -225,17 +225,6 @@
       "cursor_field": ["updatedAt"],
       "destination_sync_mode": "append"
     },
-    {
-      "stream": {
-        "name": "property_history",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "default_cursor_field": ["timestamp"]
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["timestamp"],
-      "destination_sync_mode": "overwrite"
-    },
     {
       "stream": {
         "name": "quotes",

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -1282,7 +1282,7 @@ class Owners(Stream):
     scopes = {"crm.objects.owners.read"}
 
 
-class PropertyHistory(IncrementalStream):
+class PropertyHistory(Stream):
     """Contacts Endpoint, API v1
     Is used to get all Contacts and the history of their respective
     Properties. Whenever a property is changed it is added here.
@@ -1295,9 +1295,10 @@ class PropertyHistory(IncrementalStream):
     created_at_field = "timestamp"
     entity = "contacts"
     data_field = "contacts"
-    page_field = "vid-offset"
-    page_filter = "vidOffset"
+    page_field = "time-offset"
+    page_filter = "timeOffset"
     denormalize_records = True
+    limit_field = "count"
     limit = 100
     scopes = {"crm.objects.contacts.read"}
 
@@ -1307,8 +1308,9 @@ def request_params(
         stream_slice: Mapping[str, Any] = None,
         next_page_token: Mapping[str, Any] = None,
     ) -> MutableMapping[str, Any]:
-        params = super().request_params(stream_state, stream_slice, next_page_token)
-        params.update(propertyMode="value_and_history")
+        params = {self.limit_field: self.limit, "propertyMode": "value_and_history"}
+        if next_page_token:
+            params.update(next_page_token)
         return params
 
     def _transform(self, records: Iterable) -> Iterable:

diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -129,6 +129,7 @@ Now that you have set up the HubSpot source connector, check out the following H
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.73  | 2022-07-13 | [14666](https://github.com/airbytehq/airbyte/pull/14666) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream                                                     |
 | 0.1.72  | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging                                                                                                                         |
 | 0.1.71  | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code                                                                                        | 
 | 0.1.70  | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue                                                                                                      |