Merge branch 'master' into ddavydov/#20703-source-salesforce-include-…

…pk-in-properties-chunks

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1402,6 +1402,10 @@
   icon: posthog.svg
   sourceType: api
   releaseStage: beta
+  allowedHosts:
+    hosts:
+      - "${base_url}"
+      - "app.posthog.com"
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
@@ -1918,11 +1922,14 @@
 - name: Typeform
   sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474
   dockerRepository: airbyte/source-typeform
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.com/integrations/sources/typeform
   icon: typeform.svg
   sourceType: api
   releaseStage: beta
+  allowedHosts:
+    hosts:
+      - "api.typeform.com"
 - name: US Census
   sourceDefinitionId: c4cfaeda-c757-489a-8aba-859fb08b6970
   dockerRepository: airbyte/source-us-census

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -15596,7 +15596,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-typeform:0.1.10"
+- dockerImage: "airbyte/source-typeform:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/typeform"
     connectionSpecification:

airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml

@@ -16,9 +16,13 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json"
       future_state_path: "integration_tests/future_state.json"
+      cursor_paths:
+        "events": ["2331", "timestamp"]
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json"
       future_state_path: "integration_tests/future_state_old.json"
+      cursor_paths:
+        "events": [ "2331", "timestamp" ]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-typeform/Dockerfile

@@ -12,5 +12,5 @@ COPY main.py ./
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.10
+LABEL io.airbyte.version=0.1.11
 LABEL io.airbyte.name=airbyte/source-typeform

airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml

@@ -23,10 +23,12 @@ acceptance_tests:
           path: "integration_tests/expected_records.jsonl"
   incremental:
     tests:
-      - config_path: "secrets/config.json"
+      - config_path: "secrets/incremental_config.json"
         configured_catalog_path: "integration_tests/configured_catalog_incremental.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
+        cursor_paths:
+          "responses": ["SdMKQYkv", "submitted_at"]
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

airbyte-integrations/connectors/source-typeform/source_typeform/source.py

@@ -145,6 +145,13 @@ def get_form_id(self, record: Mapping[str, Any]) -> Optional[str]:
         referer = record.get("metadata", {}).get("referer")
         return urlparse.urlparse(referer).path.split("/")[-1] if referer else None
 
+    def current_state_value_int(self, current_stream_state: MutableMapping[str, Any], form_id: str) -> int:
+        # state used to be stored as int, now we store it as str, so need to handle both cases
+        value = current_stream_state.get(form_id, {}).get(self.cursor_field, self.start_date.int_timestamp)
+        if isinstance(value, str):
+            value = pendulum.from_format(value, self.date_format).int_timestamp
+        return value
+
     def get_updated_state(
         self,
         current_stream_state: MutableMapping[str, Any],
@@ -155,10 +162,11 @@ def get_updated_state(
             return current_stream_state
 
         current_stream_state[form_id] = current_stream_state.get(form_id, {})
-        current_stream_state[form_id][self.cursor_field] = max(
+        new_state_value = max(
             pendulum.from_format(latest_record[self.cursor_field], self.date_format).int_timestamp,
-            current_stream_state[form_id].get(self.cursor_field, 1),
+            self.current_state_value_int(current_stream_state, form_id),
         )
+        current_stream_state[form_id][self.cursor_field] = pendulum.from_timestamp(new_state_value).format(self.date_format)
         return current_stream_state
 
     def request_params(
@@ -174,7 +182,7 @@ def request_params(
             # use state for first request in incremental sync
             params["sort"] = "submitted_at,asc"
             # start from last state or from start date
-            since = max(self.start_date.int_timestamp, stream_state.get(stream_slice["form_id"], {}).get(self.cursor_field, 1))
+            since = max(self.start_date.int_timestamp, self.current_state_value_int(stream_state, stream_slice["form_id"]))
             if since:
                 params["since"] = pendulum.from_timestamp(since).format(self.date_format)
         else:

airbyte-integrations/connectors/source-typeform/unit_tests/test_responses_stream.py

@@ -3,52 +3,56 @@
 #
 
 
+from typing import Any, Mapping, Optional
+
 import pendulum
 from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
 from pendulum.datetime import DateTime
 from source_typeform.source import Responses
 
-config = {"token": "10", "start_date": "2020-06-27T15:32:38Z", "page_size": 2}
+start_date_str = "2020-06-27T15:32:38Z"
+start_date = pendulum.parse(start_date_str)
+start_date_ts = start_date.int_timestamp
+config = {"token": "10", "start_date": start_date_str, "page_size": 2}
 
 UTC = pendulum.timezone("UTC")
 responses = Responses(authenticator=TokenAuthenticator(token=config["token"]), **config)
 
 
-def get_last_record(last_record_cursor: DateTime, form_id: str = "form1") -> str:
+def get_last_record(last_record_cursor: DateTime, form_id: Optional[str] = "form1") -> Mapping[str, Any]:
     metadata = {"referer": f"http://134/{form_id}"} if form_id else {}
     return {Responses.cursor_field: last_record_cursor.format(Responses.date_format), "metadata": metadata}
 
 
 def test_get_updated_state_new():
     # current record cursor greater than current state
-    current_state = {"form1": {Responses.cursor_field: 100000}}
+    current_state = {"form1": {Responses.cursor_field: start_date_ts + 100000}}
     last_record_cursor = pendulum.now(UTC)
     last_record = get_last_record(last_record_cursor)
 
     new_state = responses.get_updated_state(current_state, last_record)
-    assert new_state["form1"][Responses.cursor_field] == last_record_cursor.int_timestamp
+    assert new_state["form1"][Responses.cursor_field] == last_record_cursor.format(Responses.date_format)
 
 
 def test_get_updated_state_not_changed():
     # current record cursor less than current state
-    current_state = {"form1": {Responses.cursor_field: 100000}}
-    last_record_cursor = pendulum.from_timestamp(100)
+    current_state = {"form1": {Responses.cursor_field: start_date_ts + 100000}}
+    last_record_cursor = pendulum.from_timestamp(start_date_ts + 100)
     last_record = get_last_record(last_record_cursor)
 
     new_state = responses.get_updated_state(current_state, last_record)
-    assert new_state["form1"][Responses.cursor_field] != last_record_cursor.int_timestamp
-    assert new_state["form1"][Responses.cursor_field] == 100000
+    assert new_state["form1"][Responses.cursor_field] == pendulum.from_timestamp(start_date_ts + 100000).format(Responses.date_format)
 
 
 def test_get_updated_state_form_id_is_new():
     # current record has new form id which is not exists in current state
-    current_state = {"form1": {Responses.cursor_field: 100000}}
-    last_record_cursor = pendulum.from_timestamp(100)
+    current_state = {"form1": {Responses.cursor_field: start_date_ts + 100000}}
+    last_record_cursor = pendulum.from_timestamp(start_date_ts + 100)
     last_record = get_last_record(last_record_cursor, form_id="form2")
 
     new_state = responses.get_updated_state(current_state, last_record)
-    assert new_state["form2"][Responses.cursor_field] == last_record_cursor.int_timestamp
-    assert new_state["form1"][Responses.cursor_field] == 100000
+    assert new_state["form2"][Responses.cursor_field] == last_record_cursor.format(Responses.date_format)
+    assert new_state["form1"][Responses.cursor_field] == start_date_ts + 100000
 
 
 def test_get_updated_state_form_id_not_found_in_record():

connectors.md

@@ -240,7 +240,7 @@
 | **Twilio Taskrouter** | <img alt="Twilio Taskrouter icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/twilio.svg" height="30" height="30"/> | Source | airbyte/source-twilio-taskrouter:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/twilio-taskrouter) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twilio-taskrouter) | <small>`2446953b-b794-429b-a9b3-c821ba992a48`</small> |
 | **Twitter** | <img alt="Twitter icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/twitter.svg" height="30" height="30"/> | Source | airbyte/source-twitter:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/twitter) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twitter) | <small>`d7fd4f40-5e5a-4b8b-918f-a73077f8c131`</small> |
 | **Tyntec SMS** | <img alt="Tyntec SMS icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/tyntec.svg" height="30" height="30"/> | Source | airbyte/source-tyntec-sms:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/tyntec-sms) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tyntec-sms) | <small>`3c0c3cd1-b3e0-464a-9090-d3ceb5f92346`</small> |
-| **Typeform** | <img alt="Typeform icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/typeform.svg" height="30" height="30"/> | Source | airbyte/source-typeform:0.1.10 | beta | [link](https://docs.airbyte.com/integrations/sources/typeform) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-typeform) | <small>`e7eff203-90bf-43e5-a240-19ea3056c474`</small> |
+| **Typeform** | <img alt="Typeform icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/typeform.svg" height="30" height="30"/> | Source | airbyte/source-typeform:0.1.11 | beta | [link](https://docs.airbyte.com/integrations/sources/typeform) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-typeform) | <small>`e7eff203-90bf-43e5-a240-19ea3056c474`</small> |
 | **US Census** | <img alt="US Census icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/uscensus.svg" height="30" height="30"/> | Source | airbyte/source-us-census:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/us-census) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-us-census) | <small>`c4cfaeda-c757-489a-8aba-859fb08b6970`</small> |
 | **Vantage** | <img alt="Vantage icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/vantage.svg" height="30" height="30"/> | Source | airbyte/source-vantage:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/vantage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-vantage) | <small>`28ce1fbd-1e15-453f-aa9f-da6c4d928e92`</small> |
 | **VictorOps** | <img alt="VictorOps icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/victorops.svg" height="30" height="30"/> | Source | farosai/airbyte-victorops-source:0.1.23 | alpha | [link](https://docs.airbyte.com/integrations/sources/victorops) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/airbyte-victorops-source) | <small>`7e20ce3e-d820-4327-ad7a-88f3927fd97a`</small> |

docs/integrations/sources/typeform.md

@@ -70,6 +70,7 @@ API rate limits \(2 requests per second\): [https://developer.typeform.com/get-s
 
 | Version | Date       | Pull Request                                             | Subject                                                                 |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------|
+| 0.1.11  | 2023-02-20 | [23248](https://github.com/airbytehq/airbyte/pull/23248) | Store cursor value as a string                                          |
 | 0.1.10  | 2023-01-07 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Certification to Beta                                                   |
 | 0.1.9   | 2022-08-30 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Improve `metadata.referer` url parsing                                  |
 | 0.1.8   | 2022-08-09 | [15435](https://github.com/airbytehq/airbyte/pull/15435) | Update Forms   stream schema                                            |