#15823 source marketo: fix semi incremental streams

airbytehq · Aug 21, 2022 · e05e3a2 · e05e3a2
1 parent 9ba955d
commit e05e3a2
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 12 deletions.
diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile
@@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-marketo
diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py
@@ -83,7 +83,7 @@ def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mappin
         during the parsing.
         """
 
-        if not stream_state or record[self.cursor_field] >= stream_state.get(self.cursor_field):
+        if record[self.cursor_field] >= (stream_state or {}).get(self.cursor_field, self.start_date):
             yield record
 
     def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
@@ -136,6 +136,11 @@ def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwa
         return date_slices
 
 
+class SemiIncrementalMarketoStream(IncrementalMarketoStream):
+    def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[MutableMapping[str, any]]]:
+        return [None]
+
+
 class MarketoExportBase(IncrementalMarketoStream):
     """
     Base class for all the streams which support bulk extract.
@@ -459,14 +464,14 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str,
             yield record
 
 
-class Campaigns(IncrementalMarketoStream):
+class Campaigns(SemiIncrementalMarketoStream):
     """
     Return list of all campaigns.
     API Docs: http://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET
     """
 
 
-class Lists(IncrementalMarketoStream):
+class Lists(SemiIncrementalMarketoStream):
     """
     Return list of all lists.
     API Docs: http://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListsUsingGET

diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md
@@ -89,12 +89,13 @@ We're almost there! Armed with your Endpoint & Identity URLs and your Client ID
 
 ## CHANGELOG
 
-| Version | Date       | Pull Request                                             | Subject                                               |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------|
-| `0.1.5` | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it |
-| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs               |
-| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578)   | Updated titles and descriptions                       |
-| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483)   | Improve field conversion to conform schema            |
-| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000)   | Fix timestamp value format issue                      |
-| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863)   | Release Marketo CDK Connector                         |
+| Version | Date       | Pull Request                                             | Subject                                                                                       |
+|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------|
+| `0.1.6` | 2022-08-21 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Fix semi incremental streams: do not ignore start date, make one api call instead of multiple |
+| `0.1.5` | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it                                         |
+| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs                                                       |
+| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578)   | Updated titles and descriptions                                                               |
+| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483)   | Improve field conversion to conform schema                                                    |
+| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000)   | Fix timestamp value format issue                                                              |
+| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863)   | Release Marketo CDK Connector                                                                 |