Source Twitter: promote to cloud (#23661)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · Mar 3, 2023 · 147795c · 147795c
1 parent cea584a
commit 147795c
Show file tree

Hide file tree

Showing 12 changed files with 107 additions and 45 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -2001,11 +2001,14 @@
 - name: Twitter
   sourceDefinitionId: d7fd4f40-5e5a-4b8b-918f-a73077f8c131
   dockerRepository: airbyte/source-twitter
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.com/integrations/sources/twitter
   icon: twitter.svg
   sourceType: api
   releaseStage: alpha
+  allowedHosts:
+    hosts:
+      - api.twitter.com
 - name: Tyntec SMS
   sourceDefinitionId: 3c0c3cd1-b3e0-464a-9090-d3ceb5f92346
   dockerRepository: airbyte/source-tyntec-sms

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -15591,9 +15591,9 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-twitter:0.1.0"
+- dockerImage: "airbyte/source-twitter:0.1.1"
   spec:
-    documentationUrl: "https://docsurl.com"
+    documentationUrl: "https://docs.airbyte.com/integrations/sources/twitter"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "Twitter Spec"
@@ -15604,22 +15604,30 @@
       additionalProperties: true
       properties:
         api_key:
+          type: "string"
           title: "API Key Token"
           description: "API Key"
-          type: "string"
           airbyte_secret: true
+          order: 0
         query:
-          title: "Twitter Search Query"
-          description: "Twitter Search query"
           type: "string"
+          title: "Twitter Search Query"
+          description: "Query for matching Tweets. You can learn how to build this\
+            \ query by reading <a href=\"https://developer.twitter.com/en/docs/twitter-api/tweets/search/integrate/build-a-query\"\
+            > build a query guide </a>."
+          order: 1
         start_date:
-          title: "Start Date"
-          description: "From what date you want to start retrieving dta"
           type: "string"
+          title: "Start Date"
+          description: "The start date for retrieving tweets cannot be more than 7\
+            \ days in the past."
+          order: 2
         end_date:
-          title: "End Date"
-          description: "End data to retrieve data"
           type: "string"
+          title: "End Date"
+          description: "The end date for retrieving tweets must be a minimum of 10\
+            \ seconds prior to the request time."
+          order: 3
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-twitter/Dockerfile b/airbyte-integrations/connectors/source-twitter/Dockerfile
@@ -34,5 +34,5 @@ COPY source_twitter ./source_twitter
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-twitter
diff --git a/airbyte-integrations/connectors/source-twitter/acceptance-test-config.yml b/airbyte-integrations/connectors/source-twitter/acceptance-test-config.yml
@@ -1,6 +1,7 @@
 # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-twitter:dev
+test_strictness_level: "high"
 acceptance_tests:
   spec:
     tests:
@@ -17,21 +18,14 @@ acceptance_tests:
   basic_read:
     tests:
       - config_path: "secrets/config.json"
-        configured_catalog_path: "integration_tests/configured_catalog.json"
-        empty_streams: []
-  # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
-  #        expect_records:
-  #          path: "integration_tests/expected_records.jsonl"
-  #          extra_fields: no
-  #          exact_order: no
-  #          extra_records: yes
+        expect_records:
+          path: "integration_tests/expected_records.jsonl"
   incremental:
-    bypass_reason: "This connector does not implement incremental sync"
-  # TODO uncomment this block this block if your connector implements incremental sync:
-  #    tests:
-  #      - config_path: "secrets/config.json"
-  #        configured_catalog_path: "integration_tests/configured_catalog.json"
-  #        future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-twitter/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-twitter/integration_tests/abnormal_state.json
@@ -1,5 +1,13 @@
-{
-  "todo-stream-name": {
-    "todo-field-name": "todo-abnormal-value"
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_descriptor": {
+        "name": "tweets"
+      },
+      "stream_state": {
+        "created_at": "2099-01-01T00:00:00.000Z"
+      }
+    }
   }
-}
+]
diff --git a/airbyte-integrations/connectors/source-twitter/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-twitter/integration_tests/configured_catalog.json
@@ -4,10 +4,15 @@
       "stream": {
         "name": "tweets",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created_at"],
+        "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created_at"],
+      "primary_key": [["id"]]
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-twitter/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-twitter/integration_tests/expected_records.jsonl
@@ -0,0 +1,2 @@
+{"stream":"tweets","data":{"lang":"en","text":"RT @MichelTricot: Do you want to learn how to use #airbyte along with #dagster and #dbt? 🚀 Follow this tutorial!        https://t.co/yAunv2…","created_at":"2023-03-03T08:38:58.000Z","id":"1631574878116798465","conversation_id":"1631574878116798465","edit_history_tweet_ids":["1631574878116798465"],"author_id":"880671687930503173"},"emitted_at":1677836588279}
+{"stream":"tweets","data":{"lang":"en","text":"RT @MichelTricot: Do you want to learn how to use #airbyte along with #dagster and #dbt? 🚀 Follow this tutorial!        https://t.co/yAunv2…","created_at":"2023-03-03T02:43:06.000Z","id":"1631485318984613888","conversation_id":"1631485318984613888","edit_history_tweet_ids":["1631485318984613888"],"author_id":"1133057436720386048"},"emitted_at":1677836588282}
diff --git a/airbyte-integrations/connectors/source-twitter/source_twitter/manifest.yaml b/airbyte-integrations/connectors/source-twitter/source_twitter/manifest.yaml
@@ -13,8 +13,7 @@ definitions:
       api_token: "{{ config['api_key'] }}"
     request_parameters:
       query: "{{ config['query'] }}"
-      start_time: "{{ config['start_time'] }}"
-      end_time: "{{ config['end_time'] }}"
+      tweet.fields: "author_id,conversation_id,created_at,in_reply_to_user_id,lang"
   retriever:
     record_selector:
       $ref: "#/definitions/selector"
@@ -38,12 +37,31 @@ definitions:
   base_stream:
     retriever:
       $ref: "#/definitions/retriever"
+    incremental_sync:
+      type: DatetimeBasedCursor
+      datetime_format: "%Y-%m-%dT%H:%M:%S.000Z"
+      start_datetime:
+        datetime: "{{ config.start_date or (now_utc() - duration('P6DT23H')).strftime('%Y-%m-%dT%H:%M:%SZ') }}"
+        datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+      end_datetime:
+        datetime: "{{ config.end_date or (now_utc() - duration('PT10S')).strftime('%Y-%m-%dT%H:%M:%SZ') }}"
+        datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+      cursor_granularity: "PT0S"
+      step: "P7D"
+      start_time_option:
+        field_name: "start_time"
+        inject_into: "request_parameter"
+      end_time_option:
+        field_name: "end_time"
+        inject_into: "request_parameter"
 
   tweets_stream:
     $ref: "#/definitions/base_stream"
     $parameters:
       name: "tweets"
       path: "/search/recent"
+      cursor_field: "created_at"
+      primary_key: "id"
 
 streams:
   - "#/definitions/tweets_stream"

diff --git a/airbyte-integrations/connectors/source-twitter/source_twitter/schemas/tweets.json b/airbyte-integrations/connectors/source-twitter/source_twitter/schemas/tweets.json
@@ -3,13 +3,32 @@
   "type": "object",
   "properties": {
     "edit_history_tweet_ids": {
-      "type": ["null", "array"]
+      "type": ["null", "array"],
+      "items": {
+        "type": "string"
+      }
     },
     "id": {
       "type": ["null", "string"]
     },
     "text": {
       "type": ["null", "string"]
+    },
+    "created_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "lang": {
+      "type": ["null", "string"]
+    },
+    "author_id": {
+      "type": ["null", "string"]
+    },
+    "conversation_id": {
+      "type": ["null", "string"]
+    },
+    "in_reply_to_user_id": {
+      "type": ["null", "string"]
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-twitter/source_twitter/spec.yaml b/airbyte-integrations/connectors/source-twitter/source_twitter/spec.yaml
@@ -1,4 +1,4 @@
-documentationUrl: https://docsurl.com
+documentationUrl: https://docs.airbyte.com/integrations/sources/twitter
 connectionSpecification:
   $schema: http://json-schema.org/draft-07/schema#
   title: Twitter Spec
@@ -9,19 +9,23 @@ connectionSpecification:
   additionalProperties: true
   properties:
     api_key:
+      type: string
       title: API Key Token
       description: API Key
-      type: string
       airbyte_secret: true
+      order: 0
     query:
-      title: Twitter Search Query
-      description: Twitter Search query
       type: string
+      title: Twitter Search Query
+      description: Query for matching Tweets. You can learn how to build this query by reading <a href="https://developer.twitter.com/en/docs/twitter-api/tweets/search/integrate/build-a-query"> build a query guide </a>.
+      order: 1
     start_date:
-      title: Start Date
-      description: From what date you want to start retrieving dta
       type: string
+      title: Start Date
+      description: "The start date for retrieving tweets cannot be more than 7 days in the past."
+      order: 2
     end_date:
-      title: End Date
-      description: End data to retrieve data
       type: string
+      title: End Date
+      description: "The end date for retrieving tweets must be a minimum of 10 seconds prior to the request time."
+      order: 3
diff --git a/connectors.md b/connectors.md
@@ -239,7 +239,7 @@
 | **Trello** | <img alt="Trello icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/trello.svg" height="30" height="30"/> | Source | airbyte/source-trello:0.1.6 | alpha | [link](https://docs.airbyte.com/integrations/sources/trello) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-trello) | <small>`8da67652-004c-11ec-9a03-0242ac130003`</small> |
 | **Twilio** | <img alt="Twilio icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/twilio.svg" height="30" height="30"/> | Source | airbyte/source-twilio:0.1.16 | generally_available | [link](https://docs.airbyte.com/integrations/sources/twilio) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twilio) | <small>`b9dc6155-672e-42ea-b10d-9f1f1fb95ab1`</small> |
 | **Twilio Taskrouter** | <img alt="Twilio Taskrouter icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/twilio.svg" height="30" height="30"/> | Source | airbyte/source-twilio-taskrouter:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/twilio-taskrouter) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twilio-taskrouter) | <small>`2446953b-b794-429b-a9b3-c821ba992a48`</small> |
-| **Twitter** | <img alt="Twitter icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/twitter.svg" height="30" height="30"/> | Source | airbyte/source-twitter:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/twitter) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twitter) | <small>`d7fd4f40-5e5a-4b8b-918f-a73077f8c131`</small> |
+| **Twitter** | <img alt="Twitter icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/twitter.svg" height="30" height="30"/> | Source | airbyte/source-twitter:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/twitter) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-twitter) | <small>`d7fd4f40-5e5a-4b8b-918f-a73077f8c131`</small> |
 | **Tyntec SMS** | <img alt="Tyntec SMS icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/tyntec.svg" height="30" height="30"/> | Source | airbyte/source-tyntec-sms:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/tyntec-sms) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tyntec-sms) | <small>`3c0c3cd1-b3e0-464a-9090-d3ceb5f92346`</small> |
 | **Typeform** | <img alt="Typeform icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/typeform.svg" height="30" height="30"/> | Source | airbyte/source-typeform:0.1.11 | beta | [link](https://docs.airbyte.com/integrations/sources/typeform) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-typeform) | <small>`e7eff203-90bf-43e5-a240-19ea3056c474`</small> |
 | **US Census** | <img alt="US Census icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/uscensus.svg" height="30" height="30"/> | Source | airbyte/source-us-census:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/us-census) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-us-census) | <small>`c4cfaeda-c757-489a-8aba-859fb08b6970`</small> |

diff --git a/docs/integrations/sources/twitter.md b/docs/integrations/sources/twitter.md
@@ -8,7 +8,7 @@ The Twitter API source supports full refresh syncs
 
 Below output stream is available from this source:
 
-*[recent_search_tweets](https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-recent).
+[recent_search_tweets](https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-recent).
 
 
 ### Features
@@ -37,4 +37,5 @@ Rate limiting is mentioned in the API [docuemntation](https://developer.twitter.
 
 | Version | Date       | Pull Request                                             | Subject                                         |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------|
+| 0.1.1   | 2023-03-03 | [23661](https://github.com/airbytehq/airbyte/pull/23661) | Incremental added for the "tweets" stream       |
 | 0.1.0   | 2022-11-01 | [18883](https://github.com/airbytehq/airbyte/pull/18858) | 🎉 New Source: Twitter                          |
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"stream":"tweets","data":{"lang":"en","text":"RT @MichelTricot: Do you want to learn how to use #airbyte along with #dagster and #dbt? 🚀 Follow this tutorial! https://t.co/yAunv2…","created_at":"2023-03-03T08:38:58.000Z","id":"1631574878116798465","conversation_id":"1631574878116798465","edit_history_tweet_ids":["1631574878116798465"],"author_id":"880671687930503173"},"emitted_at":1677836588279}
		{"stream":"tweets","data":{"lang":"en","text":"RT @MichelTricot: Do you want to learn how to use #airbyte along with #dagster and #dbt? 🚀 Follow this tutorial! https://t.co/yAunv2…","created_at":"2023-03-03T02:43:06.000Z","id":"1631485318984613888","conversation_id":"1631485318984613888","edit_history_tweet_ids":["1631485318984613888"],"author_id":"1133057436720386048"},"emitted_at":1677836588282}