fixup! fixup! fixup! Attempt to add Python 3.11 support

apache · May 21, 2023 · 0642e35 · 0642e35
1 parent d6efdab
commit 0642e35
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 44 deletions.
diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml
@@ -63,7 +63,7 @@ dependencies:
   - sasl>=0.3.1; python_version>="3.9"
   - thrift>=0.9.2
 
-# Excluded because SASL is not yet compatible
+# Excluded because python-sasl is not yet compatible
 # with 3.11. See https://github.com/cloudera/python-sasl/issues/30
 excluded-python-versions:
   - "3.11"

diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/airflow/providers/apache/hive/transfers/mysql_to_hive.py
@@ -19,6 +19,7 @@
 from __future__ import annotations
 
 from collections import OrderedDict
+from contextlib import closing
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Sequence
 
@@ -131,28 +132,25 @@ def type_map(cls, mysql_type: int) -> str:
     def execute(self, context: Context):
         hive = HiveCliHook(hive_cli_conn_id=self.hive_cli_conn_id, auth=self.hive_auth)
         mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id)
-
         self.log.info("Dumping MySQL query results to local file")
-        conn = mysql.get_conn()
-        cursor = conn.cursor()
-        cursor.execute(self.sql)
         with NamedTemporaryFile("wb") as f:
-            csv_writer = csv.writer(
-                f,
-                delimiter=self.delimiter,
-                quoting=self.quoting,
-                quotechar=self.quotechar,
-                escapechar=self.escapechar,
-                encoding="utf-8",
-            )
-            field_dict = OrderedDict()
-            if cursor.description is not None:
-                for field in cursor.description:
-                    field_dict[field[0]] = self.type_map(field[1])
-            csv_writer.writerows(cursor)
+            with closing(mysql.get_conn()) as conn:
+                with closing(conn.cursor()) as cursor:
+                    cursor.execute(self.sql)
+                    csv_writer = csv.writer(
+                        f,
+                        delimiter=self.delimiter,
+                        quoting=self.quoting,
+                        quotechar=self.quotechar if self.quoting != csv.QUOTE_NONE else None,
+                        escapechar=self.escapechar,
+                        encoding="utf-8",
+                    )
+                    field_dict = OrderedDict()
+                    if cursor.description is not None:
+                        for field in cursor.description:
+                            field_dict[field[0]] = self.type_map(field[1])
+                    csv_writer.writerows(cursor)
             f.flush()
-            cursor.close()
-            conn.close()  # type: ignore[misc]
             self.log.info("Loading file into Hive")
             hive.load_file(
                 f.name,

diff --git a/pyproject.toml b/pyproject.toml
@@ -134,26 +134,6 @@ known-third-party = [
 [tool.ruff.per-file-ignores]
 "airflow/models/__init__.py" = ["F401"]
 "airflow/models/sqla_models.py" = ["F401"]
-"tests/providers/google/cloud/_internal_client/test_secret_manager_client.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_secret_manager.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_secret_manager_system.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_spanner.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_spanner.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_speech_to_text.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_speech_to_text.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_text_to_speech.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_text_to_speech.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_translate.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_translate.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_video_intelligence.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_video_intelligence.py" = ["E402"]
-"tests/providers/google/cloud/hooks/test_vision.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_vision.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_translate_speech.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_vertex_ai.py" = ["E402"]
-"tests/providers/google/cloud/operators/test_vertex_ai_system.py" = ["E402"]
-"tests/providers/google/cloud/secrets/test_secret_manager.py" = ["E402"]
-"tests/providers/google/cloud/secrets/test_secret_manager_system.py" = ["E402"]
 
 # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
 # needed for the test to work

diff --git a/setup.py b/setup.py
@@ -73,10 +73,6 @@ def fill_provider_dependencies() -> dict[str, dict[str, list[str]]]:
         )
         for key in list(dependencies.keys()):
             if CURRENT_PYTHON_VERSION in dependencies[key]["excluded-python-versions"]:
-                print(
-                    f"Excluding provider {key} because it is not "
-                    f"compatible with Python {CURRENT_PYTHON_VERSION}"
-                )
                 del dependencies[key]
         return dependencies
     except Exception as e: