feat(readiness check): enable postgres 16 support for cloudsql (#452)

feat: enable postgres 16 support for cloudsql
GoogleCloudPlatform · Jun 7, 2024 · adc798c · adc798c
1 parent f0eb7cd
commit adc798c
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 25 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -146,8 +146,8 @@
       }
   },
   "python.analysis.fixAll": [
-      "source.unusedImports",
-      "source.convertImportFormat"
+    "source.unusedImports",
+    "source.convertImportFormat"
   ],
   "sqltools.disableReleaseNotifications": true,
   "sqltools.disableNodeDetectNotifications": true,
@@ -177,5 +177,15 @@
       "tag:yaml.org,2002:python/name:material.extensions.emoji.to_svg",
       "tag:yaml.org,2002:python/name:material.extensions.emoji.twemoji",
       "tag:yaml.org,2002:python/name:pymdownx.superfences.fence_code_format"
-  ]
+],
+"python.analysis.typeCheckingMode": "standard",
+"python.analysis.inlayHints.callArgumentNames": "partial",
+"notebook.defaultFormatter": "charliermarsh.ruff",
+"editor.defaultFormatter": "charliermarsh.ruff",
+"editor.defaultFoldingRangeProvider": "charliermarsh.ruff",
+"python.languageServer": "Pylance",
+"python.analysis.experimentalserver": true,
+"python.analysis.diagnosticSeverityOverrides": {
+    "reportUnknownMemberType": "none"
+}
 }
diff --git a/pyproject.toml b/pyproject.toml
@@ -53,7 +53,7 @@ classifiers = [ # ToDo: Modify according to your needs!
 ]
 # direct dependencies of this package, installed when users `pip install dma` later.
 dependencies = [
-    "duckdb>=0.9.2",
+    "duckdb>=1.0.0",
     "polars[pyarrow]>=0.20.0",
     "aiosql>=9.4",
     "click>=8.1.7",
@@ -255,8 +255,18 @@ ignore-words-list = "alog"
 skip = 'pdm.lock, package-lock.json'
 
 [tool.pyright]
-exclude = ["scripts", "docs"]
+autoSearchPaths = false
+exclude = ["docs/**/*.py", "tests/**/*.py", "scripts/**/*.py"]
+extraPaths = []                                                # Include paths from PYTHONPATH env var and .env definition
 include = ["src/dma", "tests"]
+pythonVersion = "3.9"
+reportMissingImports = "information"
+reportMissingTypeStubs = "information"
+reportUnknownMemberType = "none"
+reportUnnecessaryTypeIgnoreComments = true
+root = ['src']
+typeCheckingMode = "standard"
+useLibraryCodeForTypes = true
 
 [tool.mypy]
 disallow_untyped_defs = false

diff --git a/src/dma/collector/workflows/readiness_check/_postgres/constants.py b/src/dma/collector/workflows/readiness_check/_postgres/constants.py
@@ -32,6 +32,7 @@
     14: "POSTGRES_14",
     15: "POSTGRES_15",
     16: "POSTGRES_16",
+    17: "POSTGRES_17",
 }
 ALLOYDB_SUPPORTED_FDWS: Final[set[str]] = {"oracle_fdw", "orafdw", "postgres_fdw"}
 CLOUDSQL_SUPPORTED_FDWS: Final[set[str]] = {"oracle_fdw", "orafdw", "postgres_fdw"}

diff --git a/src/dma/collector/workflows/readiness_check/_postgres/main.py b/src/dma/collector/workflows/readiness_check/_postgres/main.py
@@ -14,7 +14,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
 from rich.table import Table
 
@@ -70,7 +70,7 @@ class PostgresReadinessCheckTargetConfig(ReadinessCheckTargetConfig):
         db_variant="CLOUDSQL",
         minimum_supported_major_version=9.4,
         minimum_supported_rds_major_version=9.6,
-        maximum_supported_major_version=15,
+        maximum_supported_major_version=16,
         supported_collations=CLOUDSQL_SUPPORTED_COLLATIONS,
         supported_extensions=CLOUDSQL_SUPPORTED_EXTENSIONS,
         supported_fdws=CLOUDSQL_SUPPORTED_FDWS,
@@ -196,7 +196,7 @@ def _check_pglogical_privileges(self) -> list[str]:
             select has_schema_usage_privilege, has_tables_select_privilege, has_local_node_select_privilege, has_node_select_privilege,
             has_node_interface_select_privilege from collection_postgres_pglogical_privileges
         """).fetchone()
-        errors = []
+        errors: list[str] = []
         if result is None:
             errors.append("Empty result reading pglogical schema privileges for the user")
         else:
@@ -213,7 +213,7 @@ def _check_pglogical_privileges(self) -> list[str]:
         return errors
 
     def check_user_obj_privileges(self) -> list[str]:
-        errors: list = []
+        errors: list[str] = []
         rows = self.local_db.sql("""
             select namespace_name from collection_postgres_user_schemas_without_privilege
         """).fetchall()
@@ -237,7 +237,7 @@ def check_user_obj_privileges(self) -> list[str]:
 
     def _check_privileges(self, is_pglogical_installed: bool) -> None:
         rule_code = "PRIVILEGES"
-        errors = []
+        errors: list[str] = []
 
         if is_pglogical_installed:
             errors = self._check_pglogical_privileges()
@@ -263,7 +263,7 @@ def _check_wal_level(self) -> None:
             where c.setting_name='wal_level'
             and c.setting_value!='logical';
         """).fetchone()
-        wal_level = result[0] if result is not None else "unset"
+        wal_level = cast("str", result[0] if result is not None else "unset")
         for c in self.rule_config:
             if wal_level != "logical":
                 self.save_rule_result(
@@ -310,7 +310,7 @@ def _check_rds_logical_replication(self) -> None:
 
     def _check_max_replication_slots(self) -> None:
         rule_code = "MAX_REPLICATION_SLOTS"
-        url_link = "Refer https://cloud.google.com/database-migration/docs/postgres/create-migration-job#specify-source-connection-profile-info for more info."
+        url_link = "Refer to https://cloud.google.com/database-migration/docs/postgres/create-migration-job#specify-source-connection-profile-info for more info."
         db_count_result = self.local_db.sql(
             "select count(*) from extended_collection_postgres_all_databases"
         ).fetchone()
@@ -337,7 +337,7 @@ def _check_max_replication_slots(self) -> None:
                     c.db_variant,
                     rule_code,
                     "ERROR",
-                    f"Insufficient `max_replication_slots`: {total_replication_slots}, should be set to at leat {required_replication_slots}. Up to {c.extra_replication_subscriptions_required} additional subscriptions might be required depending on the parallelism level set for migration. {url_link}",
+                    f"Insufficient `max_replication_slots`: {total_replication_slots}, should be set to at least {required_replication_slots}. Up to {c.extra_replication_subscriptions_required} additional subscriptions might be required depending on the parallelism level set for migration. {url_link}",
                 )
             elif total_replication_slots < max_required_replication_slots:
                 self.save_rule_result(
@@ -356,7 +356,7 @@ def _check_max_replication_slots(self) -> None:
 
     def _check_max_wal_senders(self) -> None:
         rule_code = "MAX_WAL_SENDERS"
-        url_link = "Refer https://cloud.google.com/database-migration/docs/postgres/create-migration-job#specify-source-connection-profile-info for more info."
+        url_link = "Refer to https://cloud.google.com/database-migration/docs/postgres/create-migration-job#specify-source-connection-profile-info for more info."
         db_count_result = self.local_db.sql(
             "select count(*) from extended_collection_postgres_all_databases"
         ).fetchone()
@@ -425,7 +425,7 @@ def _check_max_wal_senders_replication_slots(self) -> None:
 
     def _check_max_worker_processes(self) -> None:
         rule_code = "MAX_WORKER_PROCESSES"
-        url_link = "Refer https://cloud.google.com/database-migration/docs/postgres/create-migration-job#specify-source-connection-profile-info for more info."
+        url_link = "Refer to https://cloud.google.com/database-migration/docs/postgres/create-migration-job#specify-source-connection-profile-info for more info."
         db_count_result = self.local_db.sql(
             "select count(*) from extended_collection_postgres_all_databases"
         ).fetchone()
@@ -488,7 +488,7 @@ def _check_fdw(self) -> None:
             from collection_postgres_table_details
             where foreign_data_wrapper_name is not null
             group by foreign_data_wrapper_name
-        """).fetchmany()
+        """).fetchall()
         fdws = {row[0] for row in result}
         fdw_table_count = {int(row[1]) for row in result}
         for c in self.rule_config:
@@ -518,10 +518,7 @@ def _print_database_details(
     ) -> None:
         """Print Summary of the Migration Readiness Assessment."""
         results = self.local_db.sql(
-            """
-                select metric_category, metric_name, metric_value
-                from collection_postgres_calculated_metrics
-            """,
+            """select metric_category, metric_name, metric_value from collection_postgres_calculated_metrics""",
         ).fetchall()
         count_table = Table(min_width=80)
         count_table.add_column("Variable Category", justify="right", style="green")
@@ -538,10 +535,8 @@ def _print_readiness_check_summary(self) -> None:
         def table_for_target(migration_target: PostgresVariants) -> None:
             results = self.local_db.execute(
                 """
-                    select severity, rule_code, info
-                    from readiness_check_summary
-                    where migration_target = ?
-                """,
+                select severity, rule_code, info from readiness_check_summary where migration_target = ?
+            """,
                 [migration_target],
             ).fetchall()
             count_table = Table(

diff --git a/src/dma/types.py b/src/dma/types.py
@@ -13,7 +13,9 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import Literal, TypeAlias
+from typing import Literal
+
+from typing_extensions import TypeAlias
 
 SupportedSources: TypeAlias = Literal["POSTGRES", "MYSQL", "ORACLE", "MSSQL"]
 PostgresVariants: TypeAlias = Literal["CLOUDSQL", "ALLOYDB"]