#291 oncall. Source Hubspot: disable Incremental mode for PropertyHis…

…tory stream and decrease number of http requests made
airbytehq · Jul 13, 2022 · 5cbba63 · 5cbba63
1 parent 48f8f2f
commit 5cbba63
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 17 deletions.
diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.72
+LABEL io.airbyte.version=0.1.73
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json
@@ -108,6 +108,15 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "property_history",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "products",

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_oauth_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_oauth_catalog.json
@@ -150,6 +150,15 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "property_history",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "subscription_changes",

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json
@@ -225,17 +225,6 @@
       "cursor_field": ["updatedAt"],
       "destination_sync_mode": "append"
     },
-    {
-      "stream": {
-        "name": "property_history",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "default_cursor_field": ["timestamp"]
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["timestamp"],
-      "destination_sync_mode": "overwrite"
-    },
     {
       "stream": {
         "name": "quotes",

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -1282,7 +1282,7 @@ class Owners(Stream):
     scopes = {"crm.objects.owners.read"}
 
 
-class PropertyHistory(IncrementalStream):
+class PropertyHistory(Stream):
     """Contacts Endpoint, API v1
     Is used to get all Contacts and the history of their respective
     Properties. Whenever a property is changed it is added here.
@@ -1295,9 +1295,10 @@ class PropertyHistory(IncrementalStream):
     created_at_field = "timestamp"
     entity = "contacts"
     data_field = "contacts"
-    page_field = "vid-offset"
-    page_filter = "vidOffset"
+    page_field = "time-offset"
+    page_filter = "timeOffset"
     denormalize_records = True
+    limit_field = "count"
     limit = 100
     scopes = {"crm.objects.contacts.read"}
 
@@ -1307,8 +1308,9 @@ def request_params(
         stream_slice: Mapping[str, Any] = None,
         next_page_token: Mapping[str, Any] = None,
     ) -> MutableMapping[str, Any]:
-        params = super().request_params(stream_state, stream_slice, next_page_token)
-        params.update(propertyMode="value_and_history")
+        params = {self.limit_field: self.limit, "propertyMode": "value_and_history"}
+        if next_page_token:
+            params.update(next_page_token)
         return params
 
     def _transform(self, records: Iterable) -> Iterable:

diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -129,6 +129,7 @@ Now that you have set up the HubSpot source connector, check out the following H
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.73  | 2022-07-13 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream                                                     |
 | 0.1.72  | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging                                                                                                                         |
 | 0.1.71  | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code                                                                                        | 
 | 0.1.70  | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue                                                                                                      |