From 4e442c4097db3cff89524b4384ccd7177dbcbbca Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 11 Jan 2024 12:11:59 +0100 Subject: [PATCH 1/5] fix stream reader --- .../connectors/source-gcs/source_gcs/stream_reader.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py index 3552d75980fd..ec44dd27048e 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py @@ -5,7 +5,6 @@ import itertools import json import logging -from contextlib import contextmanager from datetime import datetime, timedelta from io import IOBase from typing import Iterable, List, Optional @@ -94,7 +93,6 @@ def _handle_file_listing_error(self, exc: Exception, prefix: str, logger: loggin prefix=prefix, ) from exc - @contextmanager def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: """ Open and yield a remote file from GCS for reading. @@ -105,7 +103,4 @@ def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str except OSError as oe: logger.warning(ERROR_MESSAGE_ACCESS.format(uri=file.uri, bucket=self.config.bucket)) logger.exception(oe) - try: - yield result - finally: - result.close() + return result From 0f315a01706be3abec896434e5ebcded72f3b77f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 11 Jan 2024 12:13:07 +0100 Subject: [PATCH 2/5] prepare release --- airbyte-integrations/connectors/source-gcs/metadata.yaml | 2 +- docs/integrations/sources/gcs.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index e9364232fbd2..1ed2d6757783 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.3.3 + dockerImageTag: 0.3.4 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 1206c289e4f6..e94b727a05a0 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -37,6 +37,7 @@ Use the service account ID from above, grant read access to your target bucket. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| +| 0.3.4 | 2024-01-11 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Fix issue in stream reader for document file type parser | | 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | | 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | From a300c47c2f0e3be795d99a0078c003396fbd4b75 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 11 Jan 2024 12:14:40 +0100 Subject: [PATCH 3/5] fix PR number --- docs/integrations/sources/gcs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index e94b727a05a0..1dcb6d735fc0 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -37,7 +37,7 @@ Use the service account ID from above, grant read access to your target bucket. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| -| 0.3.4 | 2024-01-11 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Fix issue in stream reader for document file type parser | +| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | | 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | | 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | | 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | From 3ef2984bce96bc8a6e775f897d9765f0786ee9b5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 12 Jan 2024 09:34:05 +0100 Subject: [PATCH 4/5] fix --- .../connectors/source-gcs/source_gcs/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py index ebd1117841e1..04dc1e16b8d3 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py @@ -80,3 +80,7 @@ def replace_enum_allOf_and_anyOf(schema): objects_to_check["anyOf"] = objects_to_check.pop("allOf") return super(Config, Config).replace_enum_allOf_and_anyOf(schema) + + @staticmethod + def remove_discriminator(schema) -> None: + pass From 8980d61d864b51b77a20c8e29c0e678bdd0a6cc5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 12 Jan 2024 11:47:53 +0100 Subject: [PATCH 5/5] fix --- .../connectors/source-gcs/integration_tests/spec.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json index 560361ec3b30..5f69da41c02a 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json @@ -58,9 +58,9 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", - "type": "string", - "airbyte_hidden": true + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "airbyte_hidden": true, + "type": "string" }, "days_to_sync_if_history_is_full": { "title": "Days To Sync If History Is Full",