Fix: Resolve Python 3.9 compatibility issues, expand CI test matrix (#98

)

.github/workflows/python_pytest.yml

@@ -48,10 +48,9 @@ jobs:
  strategy:
  matrix:
  python-version: [
- # TODO: Re-enable 3.9 and 3.11 once we have stable tests across all versions.
- # '3.9',
+ '3.9',
  '3.10',
- # '3.11',
+ '3.11',
  ]
  fail-fast: false
 

airbyte/_util/document_rendering.py

@@ -2,7 +2,7 @@
 """Methods for converting Airbyte records into documents."""
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
 import yaml
 from pydantic import BaseModel
@@ -27,7 +27,7 @@ def _to_title_case(name: str, /) -> str:
 class CustomRenderingInstructions(BaseModel):
  """Instructions for rendering a stream's records as documents."""
 
- title_property: str | None
+ title_property: Optional[str]
  content_properties: list[str]
  frontmatter_properties: list[str]
  metadata_properties: list[str]
@@ -36,9 +36,9 @@ class CustomRenderingInstructions(BaseModel):
 class DocumentRenderer(BaseModel):
  """Instructions for rendering a stream's records as documents."""
 
- title_property: str | None
- content_properties: list[str] | None
- metadata_properties: list[str] | None
+ title_property: Optional[str]
+ content_properties: Optional[list[str]]
+ metadata_properties: Optional[list[str]]
  render_metadata: bool = False
 
  # TODO: Add primary key and cursor key support:

airbyte/caches/base.py

@@ -5,7 +5,7 @@
 
 import abc
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, final
+from typing import TYPE_CHECKING, Any, Optional, final
 
 from pydantic import BaseModel, PrivateAttr
 
@@ -34,7 +34,7 @@ class CacheBase(BaseModel):
  schema_name: str = "airbyte_raw"
  """The name of the schema to write to."""
 
- table_prefix: str | None = None
+ table_prefix: Optional[str] = None
  """ A prefix to add to all table names.
  If 'None', a prefix will be created based on the source name.
  """
@@ -43,7 +43,7 @@ class CacheBase(BaseModel):
  """A suffix to add to all table names."""
 
  _sql_processor_class: type[SqlProcessorBase] = PrivateAttr()
- _sql_processor: SqlProcessorBase | None = PrivateAttr(default=None)
+ _sql_processor: Optional[SqlProcessorBase] = PrivateAttr(default=None)
 
  @final
  @property

airbyte/caches/duckdb.py

@@ -5,6 +5,7 @@
 
 import warnings
 from pathlib import Path # noqa: TCH003 # Used in Pydantic init
+from typing import Union
 
 from overrides import overrides
 
@@ -23,7 +24,7 @@
 class DuckDBCache(CacheBase):
  """A DuckDB cache."""
 
- db_path: Path | str
+ db_path: Union[Path, str]
  """Normally db_path is a Path object.
 
  There are some cases, such as when connecting to MotherDuck, where it could be a string that

airbyte/documents.py

@@ -2,9 +2,9 @@
 """Methods for converting Airbyte records into documents."""
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 if TYPE_CHECKING:
@@ -30,10 +30,10 @@ class Document(BaseModel):
  This class is duck-typed to be compatible with LangChain project's `Document` class.
  """
 
- id: str | None = None
+ id: Optional[str] = Field(default=None)
  content: str
  metadata: dict[str, Any]
- last_modified: datetime.datetime | None = None
+ last_modified: Optional[datetime.datetime] = Field(default=None)
 
  def __str__(self) -> str:
  return self.content

examples/run_file_source.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""A simple test of PyAirbyte, using the File source connector.
+
+Usage (from PyAirbyte root directory):
+> poetry run python ./examples/run_file.py
+
+No setup is needed, but you may need to delete the .venv-source-file folder
+if your installation gets interrupted or corrupted.
+"""
+
+from __future__ import annotations
+
+import airbyte as ab
+
+
+source = ab.get_source(
+ "source-file",
+ install_if_missing=True,
+)
+source.check()
+
+# print(list(source.get_records("pokemon")))
+source.read(cache=ab.new_local_cache("poke"))

pyproject.toml

@@ -164,6 +164,8 @@ ignore = [
  "FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003)
  "PLW0603", # Using the global statement to update _cache is discouraged
  "TD003", # Require links for TODOs # TODO: Re-enable when we disable FIX002
+
+ "UP007", # Allow legacy `Union[a, b]` and `Optional[a]` for Pydantic, until we drop Python 3.9 (Pydantic doesn't like it)
 ]
 fixable = ["ALL"]
 unfixable = [

tests/conftest.py

@@ -1,6 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-
 """Global pytest fixtures."""
+from __future__ import annotations
 
 import json
 import logging

tests/integration_tests/fixtures/source-broken/setup.py

@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+from __future__ import annotations
 
 from setuptools import setup
 

tests/integration_tests/fixtures/source-broken/source_broken/run.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 def run():
- raise Exception("Could not run")
+ raise Exception("Could not run")

tests/integration_tests/fixtures/source-test/setup.py

@@ -1,14 +1,14 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+from __future__ import annotations
 
 from setuptools import setup
 
 setup(
  name="airbyte-source-test",
  version="0.0.1",
- description="Test Soutce",
+ description="Test Source",
  author="Airbyte",
  author_email="contact@airbyte.io",
  packages=["source_test"],

tests/integration_tests/fixtures/source-test/source_test/run.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import json
 import sys

tests/integration_tests/test_install.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from gettext import install
 import pytest

tests/integration_tests/test_snowflake_cache.py

@@ -6,6 +6,7 @@
 and available on PATH for the poetry-managed venv.
 """
 from __future__ import annotations
+
 from collections.abc import Generator
 import os
 import sys

tests/integration_tests/test_source_test_fixture.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from collections.abc import Mapping
 import os

tests/integration_tests/test_validation.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import os
 import shutil

tests/lint_tests/test_mypy.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import subprocess
 

tests/lint_tests/test_ruff.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import subprocess
 

tests/unit_tests/test_anonymous_usage_stats.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import itertools
 from contextlib import nullcontext as does_not_raise

tests/unit_tests/test_caches.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from pathlib import Path
 

tests/unit_tests/test_exceptions.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import inspect
 import pytest

tests/unit_tests/test_pip_helpers.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import pytest
 from airbyte._util import github_pip_url, connector_pip_url

tests/unit_tests/test_progress.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import datetime
 from textwrap import dedent

tests/unit_tests/test_type_translation.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import pytest
 from sqlalchemy import types