Source Gnews: reorganize inline schema in gnews.yaml

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -621,7 +621,7 @@
 - name: GNews
   sourceDefinitionId: ce38aec4-5a77-439a-be29-9ca44fd4e811
   dockerRepository: airbyte/source-gnews
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.com/integrations/sources/gnews
   icon: gnews.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4805,7 +4805,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-gnews:0.1.2"
+- dockerImage: "airbyte/source-gnews:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/gnews"
     connectionSpecification:

airbyte-integrations/connectors/source-gnews/Dockerfile

@@ -34,5 +34,5 @@ COPY source_gnews ./source_gnews
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-gnews

airbyte-integrations/connectors/source-gnews/setup.py

@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.14",
+    "airbyte-cdk~=0.19.0",
 ]
 
 TEST_REQUIREMENTS = [

airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml

@@ -1,5 +1,99 @@
 version: "0.1.0"
 
+definitions:
+  selector:
+    extractor:
+      field_pointer: ["articles"]
+  base_requester:
+    url_base: "https://gnews.io/api/v4"
+    http_method: "GET"
+    error_handler:
+      type: "CompositeErrorHandler"
+      error_handlers:
+        - response_filters:
+            - http_codes: [429]
+              action: RETRY
+          backoff_strategies:
+            - type: "ConstantBackoffStrategy"
+              backoff_time_in_seconds: 1.0
+        - response_filters:
+            - http_codes: [403]
+              action: RETRY
+          backoff_strategies:
+            - class_name: "source_gnews.WaitUntilMidnightBackoffStrategy"
+  base_retriever:
+    record_selector:
+      $ref: "*ref(definitions.selector)"
+    paginator:
+      type: NoPagination
+    stream_slicer:
+      type: "DatetimeStreamSlicer"
+      start_datetime:
+        datetime: "{{ config['start_date'] or ' '.join(day_delta(-7).split('.')[0].split('T')) }}"
+        datetime_format: "%Y-%m-%d %H:%M:%S"
+      end_datetime:
+        datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d %H:%M:%S') }}"
+        datetime_format: "%Y-%m-%d %H:%M:%S"
+      step: "P1W"
+      datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+      cursor_granularity: "PT1S"
+      cursor_field: "{{ options['stream_cursor_field'] }}"
+  common_parameters:
+    token: "{{ config['api_key'] }}"
+    lang: "{{ config['language'] }}"
+    country: "{{ config['country'] }}"
+    nullable: "{{ ','.join(config['nullable']) }}"
+    from: "{{ stream_slice['start_time'] }}"
+    to: "{{ stream_slice['end_time'] }}"
+  schema_loader:
+    type: InlineSchemaLoader
+  search_stream:
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+      schema: "*ref(schemas.search_stream_schema)"
+    $options:
+      name: "search"
+      primary_key: "url"
+      path: "/search"
+      stream_cursor_field: "publishedAt"
+    retriever:
+      $ref: "*ref(definitions.base_retriever)"
+      requester:
+        $ref: "*ref(definitions.base_requester)"
+        request_options_provider:
+          request_parameters:
+            $ref: "*ref(definitions.common_parameters)"
+            q: "{{ config['query'] }}"
+            in: "{{ ','.join(config['in']) }}"
+            sortby: "{{ config['sortby'] }}"
+  top_headlines_stream:
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+      schema: "*ref(schemas.top_headlines_stream_schema)"
+    $options:
+      name: "top_headlines"
+      primary_key: "url"
+      path: "/top-headlines"
+      stream_cursor_field: "publishedAt"
+    retriever:
+      $ref: "*ref(definitions.base_retriever)"
+      requester:
+        $ref: "*ref(definitions.base_requester)"
+        request_options_provider:
+          request_parameters:
+            $ref: "*ref(definitions.common_parameters)"
+            topic: "{{ config['top_headlines_topic'] }}"
+            q: "{{ config['top_headlines_query'] }}"
+
+streams:
+  - "*ref(definitions.search_stream)"
+  - "*ref(definitions.top_headlines_stream)"
+
+check:
+  stream_names:
+    - "search"
+    - "top_headlines"
+
 schemas:
   search_stream_schema:
     type: object
@@ -101,97 +195,3 @@ schemas:
               - "null"
               - string
             description: The home page of the source.
-
-definitions:
-  selector:
-    extractor:
-      field_pointer: ["articles"]
-  base_requester:
-    url_base: "https://gnews.io/api/v4"
-    http_method: "GET"
-    error_handler:
-      type: "CompositeErrorHandler"
-      error_handlers:
-        - response_filters:
-            - http_codes: [429]
-              action: RETRY
-          backoff_strategies:
-            - type: "ConstantBackoffStrategy"
-              backoff_time_in_seconds: 1.0
-        - response_filters:
-            - http_codes: [403]
-              action: RETRY
-          backoff_strategies:
-            - class_name: "source_gnews.WaitUntilMidnightBackoffStrategy"
-  base_retriever:
-    record_selector:
-      $ref: "*ref(definitions.selector)"
-    paginator:
-      type: NoPagination
-    stream_slicer:
-      type: "DatetimeStreamSlicer"
-      start_datetime:
-        datetime: "{{ config['start_date'] or ' '.join(day_delta(-7).split('.')[0].split('T')) }}"
-        datetime_format: "%Y-%m-%d %H:%M:%S"
-      end_datetime:
-        datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d %H:%M:%S') }}"
-        datetime_format: "%Y-%m-%d %H:%M:%S"
-      step: "P1W"
-      datetime_format: "%Y-%m-%dT%H:%M:%SZ"
-      cursor_granularity: "PT1S"
-      cursor_field: "{{ options['stream_cursor_field'] }}"
-  common_parameters:
-    token: "{{ config['api_key'] }}"
-    lang: "{{ config['language'] }}"
-    country: "{{ config['country'] }}"
-    nullable: "{{ ','.join(config['nullable']) }}"
-    from: "{{ stream_slice['start_time'] }}"
-    to: "{{ stream_slice['end_time'] }}"
-  schema_loader:
-    type: InlineSchemaLoader
-  search_stream:
-    schema_loader:
-      $ref: "*ref(definitions.schema_loader)"
-      schema: "*ref(schemas.search_stream_schema)"
-    $options:
-      name: "search"
-      primary_key: "url"
-      path: "/search"
-      stream_cursor_field: "publishedAt"
-    retriever:
-      $ref: "*ref(definitions.base_retriever)"
-      requester:
-        $ref: "*ref(definitions.base_requester)"
-        request_options_provider:
-          request_parameters:
-            $ref: "*ref(definitions.common_parameters)"
-            q: "{{ config['query'] }}"
-            in: "{{ ','.join(config['in']) }}"
-            sortby: "{{ config['sortby'] }}"
-  top_headlines_stream:
-    schema_loader:
-      $ref: "*ref(definitions.schema_loader)"
-      schema: "*ref(schemas.top_headlines_stream_schema)"
-    $options:
-      name: "top_headlines"
-      primary_key: "url"
-      path: "/top-headlines"
-      stream_cursor_field: "publishedAt"
-    retriever:
-      $ref: "*ref(definitions.base_retriever)"
-      requester:
-        $ref: "*ref(definitions.base_requester)"
-        request_options_provider:
-          request_parameters:
-            $ref: "*ref(definitions.common_parameters)"
-            topic: "{{ config['top_headlines_topic'] }}"
-            q: "{{ config['top_headlines_query'] }}"
-
-streams:
-  - "*ref(definitions.search_stream)"
-  - "*ref(definitions.top_headlines_stream)"
-
-check:
-  stream_names:
-    - "search"
-    - "top_headlines"

docs/integrations/sources/gnews.md

@@ -37,6 +37,7 @@ Rate Limiting is based on the API Key tier subscription, get more info [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                          |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------|
+| 0.1.3   | 2022-12-16 | [21322](https://github.com/airbytehq/airbyte/pull/21322) | Reorganize manifest inline stream schemas        |
 | 0.1.2   | 2022-12-16 | [20405](https://github.com/airbytehq/airbyte/pull/20405) | Update the manifest to use inline stream schemas |
 | 0.1.1   | 2022-12-13 | [20460](https://github.com/airbytehq/airbyte/pull/20460) | Update source acceptance test config             |
 | 0.1.0   | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews                             |