From e8a8cd7853c8a8d2d094182b1fa27320f2081e36 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Sun, 1 Sep 2024 23:37:02 -0700 Subject: [PATCH] refactor: rename module `_future_cdk` to `shared` --- airbyte/_processors/sql/bigquery.py | 4 ++-- airbyte/_processors/sql/duckdb.py | 4 ++-- airbyte/_processors/sql/postgres.py | 2 +- airbyte/_processors/sql/snowflake.py | 4 ++-- airbyte/_writers/base.py | 4 ++-- airbyte/_writers/file_writers.py | 4 ++-- airbyte/caches/_catalog_backend.py | 2 +- airbyte/caches/_state_backend.py | 6 +++--- airbyte/caches/_state_backend_base.py | 4 ++-- airbyte/caches/base.py | 18 +++++++++--------- airbyte/caches/snowflake.py | 2 +- airbyte/destinations/base.py | 16 ++++++++-------- airbyte/results.py | 6 +++--- airbyte/{_future_cdk => shared}/__init__.py | 2 +- .../catalog_providers.py | 0 .../{_future_cdk => shared}/sql_processor.py | 6 +++--- .../{_future_cdk => shared}/state_providers.py | 0 .../{_future_cdk => shared}/state_writers.py | 2 +- airbyte/sources/base.py | 6 +++--- .../destinations/test_source_to_destination.py | 2 +- .../test_source_test_fixture.py | 2 +- tests/integration_tests/test_state_handling.py | 2 +- 22 files changed, 49 insertions(+), 49 deletions(-) rename airbyte/{_future_cdk => shared}/__init__.py (81%) rename airbyte/{_future_cdk => shared}/catalog_providers.py (100%) rename airbyte/{_future_cdk => shared}/sql_processor.py (99%) rename airbyte/{_future_cdk => shared}/state_providers.py (100%) rename airbyte/{_future_cdk => shared}/state_writers.py (97%) diff --git a/airbyte/_processors/sql/bigquery.py b/airbyte/_processors/sql/bigquery.py index 652cc49f..c7ad7ec9 100644 --- a/airbyte/_processors/sql/bigquery.py +++ b/airbyte/_processors/sql/bigquery.py @@ -17,11 +17,11 @@ from sqlalchemy.engine import make_url from airbyte import exceptions as exc -from airbyte._future_cdk import SqlProcessorBase -from airbyte._future_cdk.sql_processor import SqlConfig from airbyte._writers.jsonl import JsonlWriter from airbyte.constants import DEFAULT_CACHE_SCHEMA_NAME from airbyte.secrets.base import SecretString +from airbyte.shared import SqlProcessorBase +from airbyte.shared.sql_processor import SqlConfig from airbyte.types import SQLTypeConverter diff --git a/airbyte/_processors/sql/duckdb.py b/airbyte/_processors/sql/duckdb.py index 0fde86d8..35162f63 100644 --- a/airbyte/_processors/sql/duckdb.py +++ b/airbyte/_processors/sql/duckdb.py @@ -12,10 +12,10 @@ from overrides import overrides from pydantic import Field -from airbyte._future_cdk import SqlProcessorBase -from airbyte._future_cdk.sql_processor import SqlConfig from airbyte._writers.jsonl import JsonlWriter from airbyte.secrets.base import SecretString +from airbyte.shared import SqlProcessorBase +from airbyte.shared.sql_processor import SqlConfig if TYPE_CHECKING: diff --git a/airbyte/_processors/sql/postgres.py b/airbyte/_processors/sql/postgres.py index ab4dbb0c..16cd5703 100644 --- a/airbyte/_processors/sql/postgres.py +++ b/airbyte/_processors/sql/postgres.py @@ -5,9 +5,9 @@ from overrides import overrides -from airbyte._future_cdk.sql_processor import SqlConfig, SqlProcessorBase from airbyte._writers.jsonl import JsonlWriter from airbyte.secrets.base import SecretString +from airbyte.shared.sql_processor import SqlConfig, SqlProcessorBase class PostgresConfig(SqlConfig): diff --git a/airbyte/_processors/sql/snowflake.py b/airbyte/_processors/sql/snowflake.py index 0da6f505..9e3493fe 100644 --- a/airbyte/_processors/sql/snowflake.py +++ b/airbyte/_processors/sql/snowflake.py @@ -14,11 +14,11 @@ from snowflake.sqlalchemy import URL, VARIANT from airbyte import exceptions as exc -from airbyte._future_cdk import SqlProcessorBase -from airbyte._future_cdk.sql_processor import SqlConfig from airbyte._writers.jsonl import JsonlWriter from airbyte.constants import DEFAULT_CACHE_SCHEMA_NAME from airbyte.secrets.base import SecretString +from airbyte.shared import SqlProcessorBase +from airbyte.shared.sql_processor import SqlConfig from airbyte.types import SQLTypeConverter diff --git a/airbyte/_writers/base.py b/airbyte/_writers/base.py index 39690058..baad82bf 100644 --- a/airbyte/_writers/base.py +++ b/airbyte/_writers/base.py @@ -8,10 +8,10 @@ if TYPE_CHECKING: - from airbyte._future_cdk.catalog_providers import CatalogProvider - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte._message_iterators import AirbyteMessageIterator from airbyte.progress import ProgressTracker + from airbyte.shared.catalog_providers import CatalogProvider + from airbyte.shared.state_writers import StateWriterBase from airbyte.strategies import WriteStrategy diff --git a/airbyte/_writers/file_writers.py b/airbyte/_writers/file_writers.py index 744379e1..b82b4a74 100644 --- a/airbyte/_writers/file_writers.py +++ b/airbyte/_writers/file_writers.py @@ -22,10 +22,10 @@ AirbyteRecordMessage, ) - from airbyte._future_cdk.catalog_providers import CatalogProvider - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte._message_iterators import AirbyteMessageIterator from airbyte.progress import ProgressTracker + from airbyte.shared.catalog_providers import CatalogProvider + from airbyte.shared.state_writers import StateWriterBase from airbyte.strategies import WriteStrategy diff --git a/airbyte/caches/_catalog_backend.py b/airbyte/caches/_catalog_backend.py index b9e1df67..b5b1adc9 100644 --- a/airbyte/caches/_catalog_backend.py +++ b/airbyte/caches/_catalog_backend.py @@ -24,7 +24,7 @@ SyncMode, ) -from airbyte._future_cdk.catalog_providers import CatalogProvider +from airbyte.shared.catalog_providers import CatalogProvider if TYPE_CHECKING: diff --git a/airbyte/caches/_state_backend.py b/airbyte/caches/_state_backend.py index ba6659ea..ac575501 100644 --- a/airbyte/caches/_state_backend.py +++ b/airbyte/caches/_state_backend.py @@ -17,18 +17,18 @@ AirbyteStateType, ) -from airbyte._future_cdk.state_providers import StaticInputState -from airbyte._future_cdk.state_writers import StateWriterBase from airbyte.caches._state_backend_base import ( StateBackendBase, ) from airbyte.exceptions import PyAirbyteInputError, PyAirbyteInternalError +from airbyte.shared.state_providers import StaticInputState +from airbyte.shared.state_writers import StateWriterBase if TYPE_CHECKING: from sqlalchemy.engine import Engine - from airbyte._future_cdk.state_providers import StateProviderBase + from airbyte.shared.state_providers import StateProviderBase CACHE_STATE_TABLE_NAME = "_airbyte_state" diff --git a/airbyte/caches/_state_backend_base.py b/airbyte/caches/_state_backend_base.py index 1212797d..899789e0 100644 --- a/airbyte/caches/_state_backend_base.py +++ b/airbyte/caches/_state_backend_base.py @@ -15,8 +15,8 @@ AirbyteStreamState, ) - from airbyte._future_cdk.state_providers import StateProviderBase - from airbyte._future_cdk.state_writers import StateWriterBase + from airbyte.shared.state_providers import StateProviderBase + from airbyte.shared.state_writers import StateWriterBase class StateBackendBase(abc.ABC): diff --git a/airbyte/caches/base.py b/airbyte/caches/base.py index 96ac559d..dfe82b73 100644 --- a/airbyte/caches/base.py +++ b/airbyte/caches/base.py @@ -13,29 +13,29 @@ from airbyte_protocol.models import ConfiguredAirbyteCatalog -from airbyte._future_cdk.catalog_providers import CatalogProvider -from airbyte._future_cdk.sql_processor import ( - SqlConfig, - SqlProcessorBase, -) -from airbyte._future_cdk.state_writers import StdOutStateWriter from airbyte._writers.base import AirbyteWriterInterface from airbyte.caches._catalog_backend import CatalogBackendBase, SqlCatalogBackend from airbyte.caches._state_backend import SqlStateBackend from airbyte.constants import DEFAULT_ARROW_MAX_CHUNK_SIZE, TEMP_FILE_CLEANUP from airbyte.datasets._sql import CachedDataset +from airbyte.shared.catalog_providers import CatalogProvider +from airbyte.shared.sql_processor import ( + SqlConfig, + SqlProcessorBase, +) +from airbyte.shared.state_writers import StdOutStateWriter if TYPE_CHECKING: from collections.abc import Iterator - from airbyte._future_cdk.sql_processor import SqlProcessorBase - from airbyte._future_cdk.state_providers import StateProviderBase - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte._message_iterators import AirbyteMessageIterator from airbyte.caches._state_backend_base import StateBackendBase from airbyte.datasets._base import DatasetBase from airbyte.progress import ProgressTracker + from airbyte.shared.sql_processor import SqlProcessorBase + from airbyte.shared.state_providers import StateProviderBase + from airbyte.shared.state_writers import StateWriterBase from airbyte.strategies import WriteStrategy diff --git a/airbyte/caches/snowflake.py b/airbyte/caches/snowflake.py index 11b6aba3..ce8831de 100644 --- a/airbyte/caches/snowflake.py +++ b/airbyte/caches/snowflake.py @@ -23,9 +23,9 @@ from pydantic import PrivateAttr -from airbyte._future_cdk.sql_processor import RecordDedupeMode from airbyte._processors.sql.snowflake import SnowflakeConfig, SnowflakeSqlProcessor from airbyte.caches.base import CacheBase +from airbyte.shared.sql_processor import RecordDedupeMode class SnowflakeCache(SnowflakeConfig, CacheBase): diff --git a/airbyte/destinations/base.py b/airbyte/destinations/base.py index 8f42362a..1b34d40f 100644 --- a/airbyte/destinations/base.py +++ b/airbyte/destinations/base.py @@ -15,27 +15,27 @@ from airbyte import exceptions as exc from airbyte._connector_base import ConnectorBase -from airbyte._future_cdk.catalog_providers import CatalogProvider -from airbyte._future_cdk.state_providers import ( - JoinedStateProvider, - StateProviderBase, - StaticInputState, -) -from airbyte._future_cdk.state_writers import NoOpStateWriter, StdOutStateWriter from airbyte._message_iterators import AirbyteMessageIterator from airbyte._util.temp_files import as_temp_files from airbyte._writers.base import AirbyteWriterInterface from airbyte.caches.util import get_default_cache from airbyte.progress import ProgressTracker from airbyte.results import ReadResult, WriteResult +from airbyte.shared.catalog_providers import CatalogProvider +from airbyte.shared.state_providers import ( + JoinedStateProvider, + StateProviderBase, + StaticInputState, +) +from airbyte.shared.state_writers import NoOpStateWriter, StdOutStateWriter from airbyte.sources.base import Source from airbyte.strategies import WriteStrategy if TYPE_CHECKING: from airbyte._executors.base import Executor - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte.caches.base import CacheBase + from airbyte.shared.state_writers import StateWriterBase class Destination(ConnectorBase, AirbyteWriterInterface): diff --git a/airbyte/results.py b/airbyte/results.py index c6a72bf0..68757304 100644 --- a/airbyte/results.py +++ b/airbyte/results.py @@ -19,13 +19,13 @@ from sqlalchemy.engine import Engine - from airbyte._future_cdk.catalog_providers import CatalogProvider - from airbyte._future_cdk.state_providers import StateProviderBase - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte._writers.base import AirbyteWriterInterface from airbyte.caches import CacheBase from airbyte.destinations.base import Destination from airbyte.progress import ProgressTracker + from airbyte.shared.catalog_providers import CatalogProvider + from airbyte.shared.state_providers import StateProviderBase + from airbyte.shared.state_writers import StateWriterBase from airbyte.sources.base import Source diff --git a/airbyte/_future_cdk/__init__.py b/airbyte/shared/__init__.py similarity index 81% rename from airbyte/_future_cdk/__init__.py rename to airbyte/shared/__init__.py index 8e612f53..00fa69a4 100644 --- a/airbyte/_future_cdk/__init__.py +++ b/airbyte/shared/__init__.py @@ -8,7 +8,7 @@ from __future__ import annotations -from airbyte._future_cdk.sql_processor import SqlProcessorBase +from airbyte.shared.sql_processor import SqlProcessorBase __all__ = [ diff --git a/airbyte/_future_cdk/catalog_providers.py b/airbyte/shared/catalog_providers.py similarity index 100% rename from airbyte/_future_cdk/catalog_providers.py rename to airbyte/shared/catalog_providers.py diff --git a/airbyte/_future_cdk/sql_processor.py b/airbyte/shared/sql_processor.py similarity index 99% rename from airbyte/_future_cdk/sql_processor.py rename to airbyte/shared/sql_processor.py index 57442912..2eac2fb2 100644 --- a/airbyte/_future_cdk/sql_processor.py +++ b/airbyte/shared/sql_processor.py @@ -41,7 +41,6 @@ ) from airbyte import exceptions as exc -from airbyte._future_cdk.state_writers import StdOutStateWriter from airbyte._util.name_normalizers import LowerCaseNormalizer from airbyte.constants import ( AB_EXTRACTED_AT_COLUMN, @@ -50,6 +49,7 @@ DEBUG_MODE, ) from airbyte.records import StreamRecordHandler +from airbyte.shared.state_writers import StdOutStateWriter from airbyte.strategies import WriteMethod, WriteStrategy from airbyte.types import SQLTypeConverter @@ -64,11 +64,11 @@ from sqlalchemy.sql.type_api import TypeEngine from airbyte._batch_handles import BatchHandle - from airbyte._future_cdk.catalog_providers import CatalogProvider - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte._writers.jsonl import FileWriterBase from airbyte.progress import ProgressTracker from airbyte.secrets.base import SecretString + from airbyte.shared.catalog_providers import CatalogProvider + from airbyte.shared.state_writers import StateWriterBase class RecordDedupeMode(enum.Enum): diff --git a/airbyte/_future_cdk/state_providers.py b/airbyte/shared/state_providers.py similarity index 100% rename from airbyte/_future_cdk/state_providers.py rename to airbyte/shared/state_providers.py diff --git a/airbyte/_future_cdk/state_writers.py b/airbyte/shared/state_writers.py similarity index 97% rename from airbyte/_future_cdk/state_writers.py rename to airbyte/shared/state_writers.py index bcc08bee..a2385d25 100644 --- a/airbyte/_future_cdk/state_writers.py +++ b/airbyte/shared/state_writers.py @@ -7,7 +7,7 @@ import abc from typing import TYPE_CHECKING, NoReturn, final -from airbyte._future_cdk.state_providers import StateProviderBase +from airbyte.shared.state_providers import StateProviderBase if TYPE_CHECKING: diff --git a/airbyte/sources/base.py b/airbyte/sources/base.py index c4493d6d..48aaece8 100644 --- a/airbyte/sources/base.py +++ b/airbyte/sources/base.py @@ -24,7 +24,6 @@ from airbyte import exceptions as exc from airbyte._connector_base import ConnectorBase -from airbyte._future_cdk.catalog_providers import CatalogProvider from airbyte._message_iterators import AirbyteMessageIterator from airbyte._util.temp_files import as_temp_files from airbyte.caches.util import get_default_cache @@ -32,6 +31,7 @@ from airbyte.progress import ProgressStyle, ProgressTracker from airbyte.records import StreamRecord, StreamRecordHandler from airbyte.results import ReadResult +from airbyte.shared.catalog_providers import CatalogProvider from airbyte.strategies import WriteStrategy @@ -42,10 +42,10 @@ from airbyte_protocol.models.airbyte_protocol import AirbyteStream from airbyte._executors.base import Executor - from airbyte._future_cdk.state_providers import StateProviderBase - from airbyte._future_cdk.state_writers import StateWriterBase from airbyte.caches import CacheBase from airbyte.documents import Document + from airbyte.shared.state_providers import StateProviderBase + from airbyte.shared.state_writers import StateWriterBase class Source(ConnectorBase): diff --git a/tests/integration_tests/destinations/test_source_to_destination.py b/tests/integration_tests/destinations/test_source_to_destination.py index 7cbc56e6..d8c25786 100644 --- a/tests/integration_tests/destinations/test_source_to_destination.py +++ b/tests/integration_tests/destinations/test_source_to_destination.py @@ -8,12 +8,12 @@ from airbyte import get_source from airbyte._executors.base import Executor from airbyte._executors.util import get_connector_executor -from airbyte._future_cdk.catalog_providers import CatalogProvider from airbyte._message_iterators import AirbyteMessageIterator from airbyte.caches.util import new_local_cache from airbyte.destinations.base import Destination from airbyte.progress import ProgressTracker from airbyte.results import ReadResult, WriteResult +from airbyte.shared.catalog_providers import CatalogProvider from airbyte.sources.base import Source from airbyte.strategies import WriteStrategy from airbyte_cdk import AirbyteMessage, AirbyteRecordMessage, Type diff --git a/tests/integration_tests/test_source_test_fixture.py b/tests/integration_tests/test_source_test_fixture.py index ce7f6597..ffc65e5f 100644 --- a/tests/integration_tests/test_source_test_fixture.py +++ b/tests/integration_tests/test_source_test_fixture.py @@ -19,12 +19,12 @@ from airbyte._executors.docker import DockerExecutor from airbyte._executors.local import PathExecutor from airbyte._executors.python import VenvExecutor -from airbyte._future_cdk.sql_processor import SqlProcessorBase from airbyte._util.venv_util import get_bin_dir from airbyte.caches import PostgresCache, SnowflakeCache from airbyte.constants import AB_INTERNAL_COLUMNS from airbyte.datasets import CachedDataset, LazyDataset, SQLDataset from airbyte.results import ReadResult +from airbyte.shared.sql_processor import SqlProcessorBase from airbyte.sources import registry from airbyte.version import get_version from sqlalchemy import column, text diff --git a/tests/integration_tests/test_state_handling.py b/tests/integration_tests/test_state_handling.py index d6056bd0..adbc288c 100644 --- a/tests/integration_tests/test_state_handling.py +++ b/tests/integration_tests/test_state_handling.py @@ -9,9 +9,9 @@ import airbyte as ab import pytest import ulid -from airbyte._future_cdk.state_writers import StateWriterBase from airbyte.caches.duckdb import DuckDBCache from airbyte.caches.util import new_local_cache +from airbyte.shared.state_writers import StateWriterBase from airbyte_protocol import models # Product count is always the same, regardless of faker scale.