🎉 Typeform - beta certification (#21145)

* fixed SAT, fix specs, updated docs * fix incremental * fix incremental * fix incremental * updated connector version in source_definitions.yaml * updated version in source_specs.yaml
airbytehq · Jan 13, 2023 · 1b185db · 1b185db
1 parent 81be661
commit 1b185db
Show file tree

Hide file tree

Showing 10 changed files with 147 additions and 113 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1821,11 +1821,11 @@
 - name: Typeform
   sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474
   dockerRepository: airbyte/source-typeform
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.10
   documentationUrl: https://docs.airbyte.com/integrations/sources/typeform
   icon: typeform.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: US Census
   sourceDefinitionId: c4cfaeda-c757-489a-8aba-859fb08b6970
   dockerRepository: airbyte/source-us-census

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -15334,7 +15334,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-typeform:0.1.9"
+- dockerImage: "airbyte/source-typeform:0.1.10"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/typeform"
     connectionSpecification:
@@ -15346,6 +15346,12 @@
       - "start_date"
       additionalProperties: true
       properties:
+        token:
+          type: "string"
+          description: "The API Token for a Typeform account."
+          title: "API Token"
+          airbyte_secret: true
+          order: 0
         start_date:
           type: "string"
           description: "UTC date and time in the format: YYYY-MM-DDTHH:mm:ss[Z]. Any\
@@ -15354,11 +15360,7 @@
           examples:
           - "2020-01-01T00:00:00Z"
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
-        token:
-          type: "string"
-          description: "The API Token for a Typeform account."
-          title: "API Token"
-          airbyte_secret: true
+          order: 1
         form_ids:
           title: "Form IDs to replicate"
           description: "When this parameter is set, the connector will replicate data\
@@ -15370,6 +15372,7 @@
           items:
             type: "string"
           uniqueItems: true
+          order: 2
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-typeform/Dockerfile b/airbyte-integrations/connectors/source-typeform/Dockerfile
@@ -12,5 +12,5 @@ COPY main.py ./
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=0.1.10
 LABEL io.airbyte.name=airbyte/source-typeform
diff --git a/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml b/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml
@@ -15,12 +15,10 @@ tests:
       empty_streams: ["webhooks"]
       expect_records:
         path: "integration_tests/expected_records.jsonl"
-  # Although incremental tests pass, they don't actually test anything because SATs currently
-  # do not support nested dynamic state. That's why incremental read is tested with a dedicated integration test.
-  #incremental:
-  #  - config_path: "secrets/config.json"
-  #    configured_catalog_path: "integration_tests/configured_catalog.json"
-  #    future_state_path: "integration_tests/abnormal_state.json"
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog_incremental.json"
+      future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json
@@ -8,6 +8,9 @@
     },
     "kRt99jlK": {
       "submitted_at": 9999999999
+    },
+    "VWO7mLtl": {
+      "submitted_at": 9999999999
     }
   }
 }
diff --git a/...grations/connectors/source-typeform/integration_tests/configured_catalog_incremental.json b/...grations/connectors/source-typeform/integration_tests/configured_catalog_incremental.json
@@ -0,0 +1,17 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "responses",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["submitted_at"],
+        "source_defined_primary_key": [["response_id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "primary_key": [["response_id"]]
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl
diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-typeform/integration_tests/test_incremental.py
@@ -6,6 +6,7 @@
 
 import pendulum
 import pytest
+import urllib.parse as urlparse
 from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type
 from source_typeform.source import SourceTypeform
 
@@ -34,7 +35,7 @@ def configured_catalog():
 def test_incremental_sync(config, configured_catalog):
     def get_form_id(record):
         referer = record.get("metadata", {}).get("referer")
-        return referer.rsplit("/")[-1] if referer else None
+        return urlparse.urlparse(referer).path.split("/")[-1] if referer else None
 
     def timestamp_from_datetime(value):
         return pendulum.from_format(value, "YYYY-MM-DDTHH:mm:ss[Z]").int_timestamp

diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json b/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json
@@ -7,18 +7,20 @@
     "required": ["token", "start_date"],
     "additionalProperties": true,
     "properties": {
+      "token": {
+        "type": "string",
+        "description": "The API Token for a Typeform account.",
+        "title": "API Token",
+        "airbyte_secret": true,
+        "order": 0
+      },
       "start_date": {
         "type": "string",
         "description": "UTC date and time in the format: YYYY-MM-DDTHH:mm:ss[Z]. Any data before this date will not be replicated.",
         "title": "Start Date",
         "examples": ["2020-01-01T00:00:00Z"],
-        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
-      },
-      "token": {
-        "type": "string",
-        "description": "The API Token for a Typeform account.",
-        "title": "API Token",
-        "airbyte_secret": true
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
+        "order": 1
       },
       "form_ids": {
         "title": "Form IDs to replicate",
@@ -27,7 +29,8 @@
         "items": {
           "type": "string"
         },
-        "uniqueItems": true
+        "uniqueItems": true,
+        "order": 2
       }
     }
   }

diff --git a/docs/integrations/sources/typeform.md b/docs/integrations/sources/typeform.md
@@ -1,49 +1,16 @@
 # Typeform
 
-## Overview
+This page guides you through the process of setting up the Typeform source connector.
 
-The Typeform Connector can be used to sync your [Typeform](https://developer.typeform.com/get-started/) data
-
-Useful links:
-
-* [Token generation](https://developer.typeform.com/get-started/personal-access-token/)
-
-#### Output schema
-
-This Source is capable of syncing the following Streams:
-
-* [Forms](https://developer.typeform.com/create/reference/retrieve-form/) \(Full Refresh\)
-* [Responses](https://developer.typeform.com/responses/reference/retrieve-responses/) \(Incremental\)
-* [Webhooks](https://developer.typeform.com/webhooks/reference/retrieve-webhooks/) \(Full Refresh\)
-* [Workspaces](https://developer.typeform.com/create/reference/retrieve-workspaces/) \(Full Refresh\)
-* [Images](https://developer.typeform.com/create/reference/retrieve-images-collection/) \(Full Refresh\)
-* [Themes](https://developer.typeform.com/create/reference/retrieve-themes/) \(Full Refresh\)
-
-#### Data type mapping
-
-| Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
-| `string`         | `string`     |       |
-| `integer`        | `integer`    |       |
-| `array`          | `array`      |       |
-| `object`         | `object`     |       |
-| `boolean`        | `boolean`    |       |
-
-#### Features
-
-| Feature                   | Supported? |
-|:--------------------------|:-----------|
-| Full Refresh Sync         | Yes        |
-| Incremental - Append Sync | Yes        |
-| Namespaces                | No         |
-
-### Requirements
+## Prerequisites
 
 * token - The Typeform API key token.
 * start\_date - Date to start fetching Responses stream data from.
 * form_ids (Optional) - List of Form Ids to sync. If not passed - sync all account`s forms.
 
-### Setup guide
+## Setup guide
+
+### Step 1: Set up Typeform
 
 To get the API token for your application follow this [steps](https://developer.typeform.com/get-started/personal-access-token/)
 
@@ -55,6 +22,39 @@ To get the API token for your application follow this [steps](https://developer.
 * Choose needed scopes \(API actions this token can perform - or permissions it has\). See here for more details on scopes.
 * Click Generate token.
 
+### Step 2: Set up the source connector in Airbyte
+
+<!-- env:cloud -->
+**For Airbyte Cloud:**
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
+3. On the source setup page, select **Typeform** from the Source type dropdown and enter a name for this connector.
+4. Fill-in 'API Token' and 'Start Date'
+5. click `Set up source`.
+<!-- /env:cloud -->
+
+<!-- env:oss -->
+**For Airbyte Open Source:**
+
+1. Go to local Airbyte page.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
+3. On the Set up the source page, enter the name for the connector and select **Tiktok Marketing** from the Source type dropdown.
+4. Fill-in 'API Token' and 'Start Date'
+5. click `Set up source`.
+<!-- /env:oss -->
+
+## Supported streams and sync modes
+
+| Stream     | Key         | Incremental | API Link                                                                    |
+|:-----------|-------------|:------------|-----------------------------------------------------------------------------|
+| Forms      | id          | No          | https://developer.typeform.com/create/reference/retrieve-form/              |
+| Responses  | response_id | Yes         | https://developer.typeform.com/responses/reference/retrieve-responses       |
+| Webhooks   | id          | No          | https://developer.typeform.com/webhooks/reference/retrieve-webhooks/        |
+| Workspaces | id          | No          | https://developer.typeform.com/create/reference/retrieve-workspaces/        |
+| Images     | id          | No          | https://developer.typeform.com/create/reference/retrieve-images-collection/ |
+| Themes     | id          | No          | https://developer.typeform.com/create/reference/retrieve-themes/            |
+
 ## Performance considerations
 
 Typeform API page size limit per source:
@@ -70,6 +70,7 @@ API rate limits \(2 requests per second\): [https://developer.typeform.com/get-s
 
 | Version | Date       | Pull Request                                             | Subject                                                                 |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------|
+| 0.1.10  | 2023-01-07 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Certification to Beta                                                   |
 | 0.1.9   | 2022-08-30 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Improve `metadata.referer` url parsing                                  |
 | 0.1.8   | 2022-08-09 | [15435](https://github.com/airbytehq/airbyte/pull/15435) | Update Forms   stream schema                                            |
 | 0.1.7   | 2022-06-20 | [13935](https://github.com/airbytehq/airbyte/pull/13935) | Update Responses stream schema                                          |