Source S3: encoding validation fix, refactor and test

airbyte-integrations/connectors/source-s3/Dockerfile

@@ -17,5 +17,5 @@ COPY source_s3 ./source_s3
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=3.1.0
+LABEL io.airbyte.version=3.1.1
 LABEL io.airbyte.name=airbyte/source-s3

airbyte-integrations/connectors/source-s3/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
-  dockerImageTag: 3.1.0
+  dockerImageTag: 3.1.1
   dockerRepository: airbyte/source-s3
   githubIssueLabel: source-s3
   icon: s3.svg

airbyte-integrations/connectors/source-s3/setup.py

@@ -9,7 +9,7 @@
     "airbyte-cdk",
     "pyarrow==9.0.0",
     "smart-open[s3]==5.1.0",
-    "wcmatch==8.2",
+    "wcmatch==8.4",
     "dill==0.3.4",
     "pytz",
     "fastavro==1.4.11",

airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py

@@ -67,6 +67,15 @@ def _validate_field(
         if field_value in disallow_values:
             return f"{field_name} can not be {field_value}"
 
+    @staticmethod
+    def _validate_encoding(encoding: str) -> None:
+        try:
+            codecs.lookup(encoding)
+        except LookupError as e:
+            # UTF8 is the default encoding value, so there is no problem if `encoding` is not set manually
+            if encoding != "":
+                raise AirbyteTracedException(str(e), str(e), failure_type=FailureType.config_error)
+
     @classmethod
     def _validate_options(cls, validator: Callable, options_name: str, format_: Mapping[str, Any]) -> Optional[str]:
         options = format_.get(options_name, "{}")
@@ -98,10 +107,7 @@ def _validate_config(self, config: Mapping[str, Any]):
             if error_message:
                 raise AirbyteTracedException(error_message, error_message, failure_type=FailureType.config_error)
 
-        try:
-            codecs.lookup(format_.get("encoding"))
-        except LookupError:
-            raise AirbyteTracedException(error_message, error_message, failure_type=FailureType.config_error)
+        self._validate_encoding(format_.get("encoding", ""))
 
     def _read_options(self) -> Mapping[str, str]:
         """

airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py

@@ -7,11 +7,14 @@
 import random
 import shutil
 import string
+from contextlib import nullcontext as does_not_raise
 from pathlib import Path
 from typing import Any, List, Mapping, Tuple
+from unittest.mock import Mock
 
 import pendulum
 import pytest
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from smart_open import open as smart_open
 from source_s3.source_files_abstract.file_info import FileInfo
 from source_s3.source_files_abstract.formats.csv_parser import CsvParser
@@ -412,3 +415,16 @@ def test_big_file(self) -> None:
                     read_count += 1
             assert read_count == expected_count
             expected_file.close()
+
+    @pytest.mark.parametrize(
+        "encoding, expectation",
+        (
+            ("UTF8", does_not_raise()),
+            ("", does_not_raise()),
+            ("R2D2", pytest.raises(AirbyteTracedException)),
+        )
+    )
+    def test_encoding_validation(self, encoding, expectation) -> None:
+        parser = CsvParser(format=Mock(), master_schema=Mock())
+        with expectation:
+            parser._validate_encoding(encoding)

docs/integrations/sources/s3.md

@@ -282,6 +282,7 @@ Be cautious when raising this value too high, as it may result in Out Of Memory
 
 | Version | Date       | Pull Request                                                                                                    | Subject                                                                                                              |
 | :------ | :--------- | :-------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- |
+| 3.1.1   | 2023-07-26 | [28730](https://github.com/airbytehq/airbyte/pull/28730)                                                        | Add human readable error message and improve validation for encoding field when it empty                             |
 | 3.1.0   | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725)                                                        | License Update: Elv2                                                                                                 |
 | 3.0.3   | 2023-06-23 | [27651](https://github.com/airbytehq/airbyte/pull/27651)                                                        | Handle Bucket Access Errors                                                                                          |
 | 3.0.2   | 2023-06-22 | [27611](https://github.com/airbytehq/airbyte/pull/27611)                                                        | Fix start date                                                                                                       |