From 4fcf946e248e5f6ab9521c0e37d73bb198deffaf Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 9 Feb 2023 10:33:36 +0200 Subject: [PATCH] #1467 source s3: review fixes --- .../formats/csv_parser.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py index 1b03aded3c1b..ac76eecfe64d 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py @@ -61,32 +61,29 @@ def _validate_field( field_value = format_.get(field_name) if not field_value and allow_empty: return - if len(format_.get(field_name)) != 1: + if len(field_value) != 1: return f"{field_name} should contain 1 character only" if field_value in disallow_values: return f"{field_name} can not be {field_value}" - @staticmethod - def _validate_read_options(format_: Mapping[str, Any]) -> Optional[str]: - options = format_.get("advanced_options", "{}") + @classmethod + def _validate_options(cls, validator: Callable, options_name: str, format_: Mapping[str, Any]) -> Optional[str]: + options = format_.get(options_name, "{}") try: options = json.loads(options) - pa.csv.ReadOptions(**options) + validator(**options) except json.decoder.JSONDecodeError: return "Malformed advanced read options!" except TypeError as e: return f"One or more read options are invalid: {str(e)}" - @staticmethod - def _validate_convert_options(format_: Mapping[str, Any]) -> Optional[str]: - options = format_.get("additional_reader_options", "{}") - try: - options = json.loads(options) - pa.csv.ConvertOptions(**options) - except json.decoder.JSONDecodeError: - return "Malformed advanced read options!" - except TypeError as e: - return f"One or more read options are invalid: {str(e)}" + @classmethod + def _validate_read_options(cls, format_: Mapping[str, Any]) -> Optional[str]: + return cls._validate_options(pa.csv.ReadOptions, "advanced_options", format_) + + @classmethod + def _validate_convert_options(cls, format_: Mapping[str, Any]) -> Optional[str]: + return cls._validate_options(pa.csv.ConvertOptions, "additional_reader_options", format_) def _validate_config(self, config: Mapping[str, Any]): format_ = config.get("format", {})