diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index e09099bd..79c0fa90 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -48,10 +48,9 @@ jobs: strategy: matrix: python-version: [ - # TODO: Re-enable 3.9 and 3.11 once we have stable tests across all versions. - # '3.9', + '3.9', '3.10', - # '3.11', + '3.11', ] fail-fast: false diff --git a/airbyte/_util/document_rendering.py b/airbyte/_util/document_rendering.py index 023a8294..322305ae 100644 --- a/airbyte/_util/document_rendering.py +++ b/airbyte/_util/document_rendering.py @@ -2,7 +2,7 @@ """Methods for converting Airbyte records into documents.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional import yaml from pydantic import BaseModel @@ -27,7 +27,7 @@ def _to_title_case(name: str, /) -> str: class CustomRenderingInstructions(BaseModel): """Instructions for rendering a stream's records as documents.""" - title_property: str | None + title_property: Optional[str] content_properties: list[str] frontmatter_properties: list[str] metadata_properties: list[str] @@ -36,9 +36,9 @@ class CustomRenderingInstructions(BaseModel): class DocumentRenderer(BaseModel): """Instructions for rendering a stream's records as documents.""" - title_property: str | None - content_properties: list[str] | None - metadata_properties: list[str] | None + title_property: Optional[str] + content_properties: Optional[list[str]] + metadata_properties: Optional[list[str]] render_metadata: bool = False # TODO: Add primary key and cursor key support: diff --git a/airbyte/caches/base.py b/airbyte/caches/base.py index 0554cf5b..f39f5c28 100644 --- a/airbyte/caches/base.py +++ b/airbyte/caches/base.py @@ -5,7 +5,7 @@ import abc from pathlib import Path -from typing import TYPE_CHECKING, Any, final +from typing import TYPE_CHECKING, Any, Optional, final from pydantic import BaseModel, PrivateAttr @@ -34,7 +34,7 @@ class CacheBase(BaseModel): schema_name: str = "airbyte_raw" """The name of the schema to write to.""" - table_prefix: str | None = None + table_prefix: Optional[str] = None """ A prefix to add to all table names. If 'None', a prefix will be created based on the source name. """ @@ -43,7 +43,7 @@ class CacheBase(BaseModel): """A suffix to add to all table names.""" _sql_processor_class: type[SqlProcessorBase] = PrivateAttr() - _sql_processor: SqlProcessorBase | None = PrivateAttr(default=None) + _sql_processor: Optional[SqlProcessorBase] = PrivateAttr(default=None) @final @property diff --git a/airbyte/caches/duckdb.py b/airbyte/caches/duckdb.py index 4d396cf9..91693ac5 100644 --- a/airbyte/caches/duckdb.py +++ b/airbyte/caches/duckdb.py @@ -5,6 +5,7 @@ import warnings from pathlib import Path # noqa: TCH003 # Used in Pydantic init +from typing import Union from overrides import overrides @@ -23,7 +24,7 @@ class DuckDBCache(CacheBase): """A DuckDB cache.""" - db_path: Path | str + db_path: Union[Path, str] """Normally db_path is a Path object. There are some cases, such as when connecting to MotherDuck, where it could be a string that diff --git a/airbyte/documents.py b/airbyte/documents.py index 8af80673..30ef0f3a 100644 --- a/airbyte/documents.py +++ b/airbyte/documents.py @@ -2,9 +2,9 @@ """Methods for converting Airbyte records into documents.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field if TYPE_CHECKING: @@ -30,10 +30,10 @@ class Document(BaseModel): This class is duck-typed to be compatible with LangChain project's `Document` class. """ - id: str | None = None + id: Optional[str] = Field(default=None) content: str metadata: dict[str, Any] - last_modified: datetime.datetime | None = None + last_modified: Optional[datetime.datetime] = Field(default=None) def __str__(self) -> str: return self.content diff --git a/examples/run_file_source.py b/examples/run_file_source.py new file mode 100644 index 00000000..a264a33e --- /dev/null +++ b/examples/run_file_source.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""A simple test of PyAirbyte, using the File source connector. + +Usage (from PyAirbyte root directory): +> poetry run python ./examples/run_file.py + +No setup is needed, but you may need to delete the .venv-source-file folder +if your installation gets interrupted or corrupted. +""" + +from __future__ import annotations + +import airbyte as ab + + +source = ab.get_source( + "source-file", + install_if_missing=True, +) +source.check() + +# print(list(source.get_records("pokemon"))) +source.read(cache=ab.new_local_cache("poke")) diff --git a/pyproject.toml b/pyproject.toml index 0a6a7cd6..bedc8d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,6 +164,8 @@ ignore = [ "FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003) "PLW0603", # Using the global statement to update _cache is discouraged "TD003", # Require links for TODOs # TODO: Re-enable when we disable FIX002 + + "UP007", # Allow legacy `Union[a, b]` and `Optional[a]` for Pydantic, until we drop Python 3.9 (Pydantic doesn't like it) ] fixable = ["ALL"] unfixable = [ diff --git a/tests/conftest.py b/tests/conftest.py index 73b2c763..7d4c4854 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - """Global pytest fixtures.""" +from __future__ import annotations import json import logging diff --git a/tests/integration_tests/fixtures/source-broken/setup.py b/tests/integration_tests/fixtures/source-broken/setup.py index 51611271..77f41cae 100644 --- a/tests/integration_tests/fixtures/source-broken/setup.py +++ b/tests/integration_tests/fixtures/source-broken/setup.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +from __future__ import annotations from setuptools import setup diff --git a/tests/integration_tests/fixtures/source-broken/source_broken/run.py b/tests/integration_tests/fixtures/source-broken/source_broken/run.py index c777271f..0e8e19f9 100644 --- a/tests/integration_tests/fixtures/source-broken/source_broken/run.py +++ b/tests/integration_tests/fixtures/source-broken/source_broken/run.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations def run(): - raise Exception("Could not run") \ No newline at end of file + raise Exception("Could not run") diff --git a/tests/integration_tests/fixtures/source-test/setup.py b/tests/integration_tests/fixtures/source-test/setup.py index 0035f1ed..caf5b94f 100644 --- a/tests/integration_tests/fixtures/source-test/setup.py +++ b/tests/integration_tests/fixtures/source-test/setup.py @@ -1,14 +1,14 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +from __future__ import annotations from setuptools import setup setup( name="airbyte-source-test", version="0.0.1", - description="Test Soutce", + description="Test Source", author="Airbyte", author_email="contact@airbyte.io", packages=["source_test"], diff --git a/tests/integration_tests/fixtures/source-test/source_test/run.py b/tests/integration_tests/fixtures/source-test/source_test/run.py index 5f4ae3f1..1d6c4820 100644 --- a/tests/integration_tests/fixtures/source-test/source_test/run.py +++ b/tests/integration_tests/fixtures/source-test/source_test/run.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import json import sys diff --git a/tests/integration_tests/test_install.py b/tests/integration_tests/test_install.py index 65eb5593..fc5766e5 100644 --- a/tests/integration_tests/test_install.py +++ b/tests/integration_tests/test_install.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from gettext import install import pytest diff --git a/tests/integration_tests/test_snowflake_cache.py b/tests/integration_tests/test_snowflake_cache.py index 7c109dc8..1e3aabad 100644 --- a/tests/integration_tests/test_snowflake_cache.py +++ b/tests/integration_tests/test_snowflake_cache.py @@ -6,6 +6,7 @@ and available on PATH for the poetry-managed venv. """ from __future__ import annotations + from collections.abc import Generator import os import sys diff --git a/tests/integration_tests/test_source_test_fixture.py b/tests/integration_tests/test_source_test_fixture.py index d1e5083f..c8b04426 100644 --- a/tests/integration_tests/test_source_test_fixture.py +++ b/tests/integration_tests/test_source_test_fixture.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from collections.abc import Mapping import os diff --git a/tests/integration_tests/test_validation.py b/tests/integration_tests/test_validation.py index dd7cf186..273dd21f 100644 --- a/tests/integration_tests/test_validation.py +++ b/tests/integration_tests/test_validation.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import os import shutil diff --git a/tests/lint_tests/test_mypy.py b/tests/lint_tests/test_mypy.py index df099782..48b2c1b8 100644 --- a/tests/lint_tests/test_mypy.py +++ b/tests/lint_tests/test_mypy.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import subprocess diff --git a/tests/lint_tests/test_ruff.py b/tests/lint_tests/test_ruff.py index 57262a8f..69894cf4 100644 --- a/tests/lint_tests/test_ruff.py +++ b/tests/lint_tests/test_ruff.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import subprocess diff --git a/tests/unit_tests/test_anonymous_usage_stats.py b/tests/unit_tests/test_anonymous_usage_stats.py index 40f52018..b1b953e8 100644 --- a/tests/unit_tests/test_anonymous_usage_stats.py +++ b/tests/unit_tests/test_anonymous_usage_stats.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import itertools from contextlib import nullcontext as does_not_raise diff --git a/tests/unit_tests/test_caches.py b/tests/unit_tests/test_caches.py index fcfb3239..6f676018 100644 --- a/tests/unit_tests/test_caches.py +++ b/tests/unit_tests/test_caches.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations from pathlib import Path diff --git a/tests/unit_tests/test_exceptions.py b/tests/unit_tests/test_exceptions.py index f2e53955..919ea7ed 100644 --- a/tests/unit_tests/test_exceptions.py +++ b/tests/unit_tests/test_exceptions.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import inspect import pytest diff --git a/tests/unit_tests/test_pip_helpers.py b/tests/unit_tests/test_pip_helpers.py index dfac7294..ba790e9a 100644 --- a/tests/unit_tests/test_pip_helpers.py +++ b/tests/unit_tests/test_pip_helpers.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import pytest from airbyte._util import github_pip_url, connector_pip_url diff --git a/tests/unit_tests/test_progress.py b/tests/unit_tests/test_progress.py index d88cbf91..3baadb5f 100644 --- a/tests/unit_tests/test_progress.py +++ b/tests/unit_tests/test_progress.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import datetime from textwrap import dedent diff --git a/tests/unit_tests/test_type_translation.py b/tests/unit_tests/test_type_translation.py index 684cdc10..69271b10 100644 --- a/tests/unit_tests/test_type_translation.py +++ b/tests/unit_tests/test_type_translation.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations import pytest from sqlalchemy import types