From b19938e806af46490bc8c1a7aa32fe30e6ec029e Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 12 Jan 2023 15:07:48 +0000 Subject: [PATCH 1/2] [source-gnews] Reorganize gnews.yaml inline schemas --- .../source-gnews/source_gnews/gnews.yaml | 188 +++++++++--------- docs/integrations/sources/gnews.md | 1 + 2 files changed, 95 insertions(+), 94 deletions(-) diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml b/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml index cd7c642eb5fa..e00aba0ce5e8 100644 --- a/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml +++ b/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml @@ -1,5 +1,99 @@ version: "0.1.0" +definitions: + selector: + extractor: + field_pointer: ["articles"] + base_requester: + url_base: "https://gnews.io/api/v4" + http_method: "GET" + error_handler: + type: "CompositeErrorHandler" + error_handlers: + - response_filters: + - http_codes: [429] + action: RETRY + backoff_strategies: + - type: "ConstantBackoffStrategy" + backoff_time_in_seconds: 1.0 + - response_filters: + - http_codes: [403] + action: RETRY + backoff_strategies: + - class_name: "source_gnews.WaitUntilMidnightBackoffStrategy" + base_retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_date'] or ' '.join(day_delta(-7).split('.')[0].split('T')) }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + end_datetime: + datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d %H:%M:%S') }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + step: "P1W" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: "PT1S" + cursor_field: "{{ options['stream_cursor_field'] }}" + common_parameters: + token: "{{ config['api_key'] }}" + lang: "{{ config['language'] }}" + country: "{{ config['country'] }}" + nullable: "{{ ','.join(config['nullable']) }}" + from: "{{ stream_slice['start_time'] }}" + to: "{{ stream_slice['end_time'] }}" + schema_loader: + type: InlineSchemaLoader + search_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + schema: "*ref(schemas.search_stream_schema)" + $options: + name: "search" + primary_key: "url" + path: "/search" + stream_cursor_field: "publishedAt" + retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.base_requester)" + request_options_provider: + request_parameters: + $ref: "*ref(definitions.common_parameters)" + q: "{{ config['query'] }}" + in: "{{ ','.join(config['in']) }}" + sortby: "{{ config['sortby'] }}" + top_headlines_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + schema: "*ref(schemas.top_headlines_stream_schema)" + $options: + name: "top_headlines" + primary_key: "url" + path: "/top-headlines" + stream_cursor_field: "publishedAt" + retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.base_requester)" + request_options_provider: + request_parameters: + $ref: "*ref(definitions.common_parameters)" + topic: "{{ config['top_headlines_topic'] }}" + q: "{{ config['top_headlines_query'] }}" + +streams: + - "*ref(definitions.search_stream)" + - "*ref(definitions.top_headlines_stream)" + +check: + stream_names: + - "search" + - "top_headlines" + schemas: search_stream_schema: type: object @@ -101,97 +195,3 @@ schemas: - "null" - string description: The home page of the source. - -definitions: - selector: - extractor: - field_pointer: ["articles"] - base_requester: - url_base: "https://gnews.io/api/v4" - http_method: "GET" - error_handler: - type: "CompositeErrorHandler" - error_handlers: - - response_filters: - - http_codes: [429] - action: RETRY - backoff_strategies: - - type: "ConstantBackoffStrategy" - backoff_time_in_seconds: 1.0 - - response_filters: - - http_codes: [403] - action: RETRY - backoff_strategies: - - class_name: "source_gnews.WaitUntilMidnightBackoffStrategy" - base_retriever: - record_selector: - $ref: "*ref(definitions.selector)" - paginator: - type: NoPagination - stream_slicer: - type: "DatetimeStreamSlicer" - start_datetime: - datetime: "{{ config['start_date'] or ' '.join(day_delta(-7).split('.')[0].split('T')) }}" - datetime_format: "%Y-%m-%d %H:%M:%S" - end_datetime: - datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d %H:%M:%S') }}" - datetime_format: "%Y-%m-%d %H:%M:%S" - step: "P1W" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - cursor_granularity: "PT1S" - cursor_field: "{{ options['stream_cursor_field'] }}" - common_parameters: - token: "{{ config['api_key'] }}" - lang: "{{ config['language'] }}" - country: "{{ config['country'] }}" - nullable: "{{ ','.join(config['nullable']) }}" - from: "{{ stream_slice['start_time'] }}" - to: "{{ stream_slice['end_time'] }}" - schema_loader: - type: InlineSchemaLoader - search_stream: - schema_loader: - $ref: "*ref(definitions.schema_loader)" - schema: "*ref(schemas.search_stream_schema)" - $options: - name: "search" - primary_key: "url" - path: "/search" - stream_cursor_field: "publishedAt" - retriever: - $ref: "*ref(definitions.base_retriever)" - requester: - $ref: "*ref(definitions.base_requester)" - request_options_provider: - request_parameters: - $ref: "*ref(definitions.common_parameters)" - q: "{{ config['query'] }}" - in: "{{ ','.join(config['in']) }}" - sortby: "{{ config['sortby'] }}" - top_headlines_stream: - schema_loader: - $ref: "*ref(definitions.schema_loader)" - schema: "*ref(schemas.top_headlines_stream_schema)" - $options: - name: "top_headlines" - primary_key: "url" - path: "/top-headlines" - stream_cursor_field: "publishedAt" - retriever: - $ref: "*ref(definitions.base_retriever)" - requester: - $ref: "*ref(definitions.base_requester)" - request_options_provider: - request_parameters: - $ref: "*ref(definitions.common_parameters)" - topic: "{{ config['top_headlines_topic'] }}" - q: "{{ config['top_headlines_query'] }}" - -streams: - - "*ref(definitions.search_stream)" - - "*ref(definitions.top_headlines_stream)" - -check: - stream_names: - - "search" - - "top_headlines" diff --git a/docs/integrations/sources/gnews.md b/docs/integrations/sources/gnews.md index 39cf3c983ab8..3244e320d520 100644 --- a/docs/integrations/sources/gnews.md +++ b/docs/integrations/sources/gnews.md @@ -37,6 +37,7 @@ Rate Limiting is based on the API Key tier subscription, get more info [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------| +| 0.1.2 | 2022-12-16 | [21322](https://github.com/airbytehq/airbyte/pull/21322) | Reorganize manifest inline stream schemas | | 0.1.2 | 2022-12-16 | [20405](https://github.com/airbytehq/airbyte/pull/20405) | Update the manifest to use inline stream schemas | | 0.1.1 | 2022-12-13 | [20460](https://github.com/airbytehq/airbyte/pull/20460) | Update source acceptance test config | | 0.1.0 | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews | From f093d95aaacb7aa77dbc929f5444d047b716dae9 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Thu, 12 Jan 2023 11:27:55 -0500 Subject: [PATCH 2/2] Bump gnews.md version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- airbyte-integrations/connectors/source-gnews/Dockerfile | 2 +- airbyte-integrations/connectors/source-gnews/setup.py | 2 +- docs/integrations/sources/gnews.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5289de9cd9dd..da2a3bf59d42 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -611,7 +611,7 @@ - name: GNews sourceDefinitionId: ce38aec4-5a77-439a-be29-9ca44fd4e811 dockerRepository: airbyte/source-gnews - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/sources/gnews icon: gnews.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 2fcc63834bf0..ab13be9829ae 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4781,7 +4781,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-gnews:0.1.2" +- dockerImage: "airbyte/source-gnews:0.1.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/gnews" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-gnews/Dockerfile b/airbyte-integrations/connectors/source-gnews/Dockerfile index fc9c255074f3..bf02171b1048 100644 --- a/airbyte-integrations/connectors/source-gnews/Dockerfile +++ b/airbyte-integrations/connectors/source-gnews/Dockerfile @@ -34,5 +34,5 @@ COPY source_gnews ./source_gnews ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-gnews diff --git a/airbyte-integrations/connectors/source-gnews/setup.py b/airbyte-integrations/connectors/source-gnews/setup.py index 23ca1049cc0e..96dd86e05988 100644 --- a/airbyte-integrations/connectors/source-gnews/setup.py +++ b/airbyte-integrations/connectors/source-gnews/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.14", + "airbyte-cdk~=0.19.0", ] TEST_REQUIREMENTS = [ diff --git a/docs/integrations/sources/gnews.md b/docs/integrations/sources/gnews.md index 3244e320d520..ebbb9e45812d 100644 --- a/docs/integrations/sources/gnews.md +++ b/docs/integrations/sources/gnews.md @@ -37,7 +37,7 @@ Rate Limiting is based on the API Key tier subscription, get more info [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------| -| 0.1.2 | 2022-12-16 | [21322](https://github.com/airbytehq/airbyte/pull/21322) | Reorganize manifest inline stream schemas | +| 0.1.3 | 2022-12-16 | [21322](https://github.com/airbytehq/airbyte/pull/21322) | Reorganize manifest inline stream schemas | | 0.1.2 | 2022-12-16 | [20405](https://github.com/airbytehq/airbyte/pull/20405) | Update the manifest to use inline stream schemas | | 0.1.1 | 2022-12-13 | [20460](https://github.com/airbytehq/airbyte/pull/20460) | Update source acceptance test config | | 0.1.0 | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews |