diff --git a/airbyte/_processors/sql/bigquery.py b/airbyte/_processors/sql/bigquery.py index 3458c53a..182c72a8 100644 --- a/airbyte/_processors/sql/bigquery.py +++ b/airbyte/_processors/sql/bigquery.py @@ -3,6 +3,7 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import TYPE_CHECKING, Optional, final @@ -56,7 +57,17 @@ def dataset_name(self) -> str: @overrides def get_sql_alchemy_url(self) -> SecretString: - """Return the SQLAlchemy URL to use.""" + """Return the SQLAlchemy URL to use. + + We suppress warnings about unrecognized JSON type. More info on that here: + - https://github.com/airbytehq/PyAirbyte/issues/254 + """ + warnings.filterwarnings( + "ignore", + message="Did not recognize type 'JSON' of column", + category=sqlalchemy.exc.SAWarning, + ) + url: URL = make_url(f"bigquery://{self.project_name!s}") if self.credentials_path: url = url.update_query_dict({"credentials_path": self.credentials_path}) diff --git a/airbyte/_processors/sql/duckdb.py b/airbyte/_processors/sql/duckdb.py index 2257102d..38b3a2e9 100644 --- a/airbyte/_processors/sql/duckdb.py +++ b/airbyte/_processors/sql/duckdb.py @@ -8,6 +8,7 @@ from textwrap import dedent, indent from typing import TYPE_CHECKING, Union +from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field from typing_extensions import Literal @@ -22,14 +23,6 @@ from sqlalchemy.engine import Engine -# Suppress warnings from DuckDB about reflection on indices. -# https://github.com/Mause/duckdb_engine/issues/905 -warnings.filterwarnings( - "ignore", - message="duckdb-engine doesn't yet support reflection on indices", -) - - # @dataclass class DuckDBConfig(SqlConfig): """Configuration for DuckDB.""" @@ -47,7 +40,13 @@ class DuckDBConfig(SqlConfig): @overrides def get_sql_alchemy_url(self) -> SecretString: """Return the SQLAlchemy URL to use.""" - # return f"duckdb:///{self.db_path}?schema={self.schema_name}" + # Suppress warnings from DuckDB about reflection on indices. + # https://github.com/Mause/duckdb_engine/issues/905 + warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, + ) return SecretString(f"duckdb:///{self.db_path!s}") @overrides diff --git a/airbyte/_processors/sql/motherduck.py b/airbyte/_processors/sql/motherduck.py index 37e79352..edece91e 100644 --- a/airbyte/_processors/sql/motherduck.py +++ b/airbyte/_processors/sql/motherduck.py @@ -6,6 +6,7 @@ import warnings from typing import TYPE_CHECKING +from duckdb_engine import DuckDBEngineWarning from overrides import overrides from airbyte._processors.file import JsonlWriter @@ -21,6 +22,7 @@ warnings.filterwarnings( "ignore", message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, ) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index 845d6742..e1e1e91f 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -5,6 +5,8 @@ import json import tempfile +import time +import warnings from contextlib import contextmanager, suppress from pathlib import Path from typing import TYPE_CHECKING, Any @@ -20,18 +22,38 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, temp_files: list[Any] = [] try: for content in files_contents: + use_json = isinstance(content, dict) temp_file = tempfile.NamedTemporaryFile( mode="w+t", delete=False, encoding="utf-8", + suffix=".json" if use_json else ".txt", ) temp_file.write( json.dumps(content) if isinstance(content, dict) else content, ) temp_file.flush() + # Don't close the file yet (breaks Windows) + # temp_file.close() temp_files.append(temp_file) yield [file.name for file in temp_files] finally: for temp_file in temp_files: - with suppress(Exception): - Path(temp_file.name).unlink() + max_attempts = 5 + for attempt in range(max_attempts): + try: + with suppress(Exception): + temp_file.close() + + Path(temp_file.name).unlink(missing_ok=True) + + break # File was deleted successfully. Move on. + except Exception as ex: + if attempt < max_attempts - 1: + time.sleep(1) # File might not be closed yet. Wait and try again. + else: + # Something went wrong and the file could not be deleted. Warn the user. + warnings.warn( + f"Failed to remove temporary file: '{temp_file.name}'. {ex}", + stacklevel=2, + ) diff --git a/airbyte/caches/duckdb.py b/airbyte/caches/duckdb.py index 12eb4225..47b56020 100644 --- a/airbyte/caches/duckdb.py +++ b/airbyte/caches/duckdb.py @@ -18,6 +18,7 @@ import warnings +from duckdb_engine import DuckDBEngineWarning from pydantic import PrivateAttr from airbyte._processors.sql.duckdb import DuckDBConfig, DuckDBSqlProcessor @@ -29,6 +30,7 @@ warnings.filterwarnings( "ignore", message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, ) diff --git a/airbyte/caches/motherduck.py b/airbyte/caches/motherduck.py index f6dfd2cb..62a140e5 100644 --- a/airbyte/caches/motherduck.py +++ b/airbyte/caches/motherduck.py @@ -16,6 +16,9 @@ from __future__ import annotations +import warnings + +from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field, PrivateAttr @@ -35,6 +38,14 @@ class MotherDuckConfig(DuckDBConfig): @overrides def get_sql_alchemy_url(self) -> SecretString: """Return the SQLAlchemy URL to use.""" + # Suppress warnings from DuckDB about reflection on indices. + # https://github.com/Mause/duckdb_engine/issues/905 + warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, + ) + return SecretString( f"duckdb:///md:{self.database}?motherduck_token={self.api_key}" # f"&schema={self.schema_name}" # TODO: Debug why this doesn't work diff --git a/airbyte/cloud/experimental.py b/airbyte/cloud/experimental.py index fbc3ace4..91bd6223 100644 --- a/airbyte/cloud/experimental.py +++ b/airbyte/cloud/experimental.py @@ -27,6 +27,7 @@ import warnings +from airbyte import exceptions as exc from airbyte.cloud.connections import CloudConnection as Stable_CloudConnection from airbyte.cloud.workspaces import CloudWorkspace as Stable_CloudWorkspace @@ -35,7 +36,7 @@ # explicitly imports it. warnings.warn( message="The `airbyte.cloud.experimental` module is experimental and may change in the future.", - category=FutureWarning, + category=exc.AirbyteExperimentalFeatureWarning, stacklevel=2, ) diff --git a/airbyte/cloud/sync_results.py b/airbyte/cloud/sync_results.py index 997566e7..89beab15 100644 --- a/airbyte/cloud/sync_results.py +++ b/airbyte/cloud/sync_results.py @@ -294,8 +294,16 @@ def get_dataset(self, stream_name: str) -> CachedDataset: """Retrieve an `airbyte.datasets.CachedDataset` object for a given stream name. This can be used to read and analyze the data in a SQL-based destination. + + TODO: In a future iteration, we can consider providing stream configuration information + (catalog information) to the `CachedDataset` object via the "Get stream properties" + API: https://reference.airbyte.com/reference/getstreamproperties """ - return CachedDataset(self.get_sql_cache(), stream_name=stream_name) + return CachedDataset( + self.get_sql_cache(), + stream_name=stream_name, + stream_configuration=False, # Don't look for stream configuration in cache. + ) def get_sql_database_name(self) -> str: """Return the SQL database name.""" diff --git a/airbyte/datasets/_sql.py b/airbyte/datasets/_sql.py index a5d36b29..b5e310d5 100644 --- a/airbyte/datasets/_sql.py +++ b/airbyte/datasets/_sql.py @@ -139,10 +139,17 @@ def __init__( self, cache: CacheBase, stream_name: str, + stream_configuration: ConfiguredAirbyteStream | None | Literal[False] = None, ) -> None: """We construct the query statement by selecting all columns from the table. This prevents the need to scan the table schema to construct the query statement. + + If stream_configuration is None, we attempt to retrieve the stream configuration from the + cache processor. This is useful when constructing a dataset from a CachedDataset object, + which already has the stream configuration. + + If stream_configuration is set to False, we skip the stream configuration retrieval. """ table_name = cache.processor.get_sql_table_name(stream_name) schema_name = cache.schema_name @@ -151,6 +158,7 @@ def __init__( cache=cache, stream_name=stream_name, query_statement=query, + stream_configuration=stream_configuration, ) @overrides diff --git a/airbyte/exceptions.py b/airbyte/exceptions.py index b3b4f367..cd084446 100644 --- a/airbyte/exceptions.py +++ b/airbyte/exceptions.py @@ -397,3 +397,10 @@ class AirbyteMultipleResourcesError(AirbyteError): resource_type: str | None = None resource_name_or_id: str | None = None + + +# Custom Warnings + + +class AirbyteExperimentalFeatureWarning(FutureWarning): + """Warning whenever using experimental features in PyAirbyte.""" diff --git a/airbyte/validate.py b/airbyte/validate.py index 4ecf2d1c..31a87f5e 100644 --- a/airbyte/validate.py +++ b/airbyte/validate.py @@ -63,7 +63,7 @@ def full_tests(connector_name: str, sample_config: str) -> None: source = ab.get_source( # TODO: FIXME: noqa: SIM115, PTH123 connector_name, - config=json.load(open(sample_config, encoding="utf-8")), # noqa: SIM115, PTH123, + config=json.loads(Path(sample_config).read_text(encoding="utf-8")), # , install_if_missing=False, ) @@ -118,8 +118,7 @@ def run() -> None: def validate(connector_dir: str, sample_config: str, *, validate_install_only: bool) -> None: # read metadata.yaml metadata_path = Path(connector_dir) / "metadata.yaml" - with Path(metadata_path).open(encoding="utf-8") as stream: - metadata = yaml.safe_load(stream)["data"] + metadata = yaml.safe_load(Path(metadata_path).read_text(encoding="utf-8"))["data"] # TODO: Use remoteRegistries.pypi.packageName once set for connectors connector_name = metadata["dockerRepository"].replace("airbyte/", "") @@ -147,15 +146,24 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b ], } - with tempfile.NamedTemporaryFile(mode="w+t", delete=True, encoding="utf-8") as temp_file: + with tempfile.NamedTemporaryFile( + mode="w+t", delete=True, encoding="utf-8", suffix="-catalog.json" + ) as temp_file: temp_file.write(json.dumps(registry)) - temp_file.seek(0) + temp_file.flush() os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name) - if validate_install_only: - install_only_test(connector_name) - else: - if not sample_config: - raise exc.PyAirbyteInputError( - input_value="--sample-config is required without --validate-install-only set" - ) - full_tests(connector_name, sample_config) + try: + if validate_install_only: + install_only_test(connector_name) + else: + if not sample_config: + raise exc.PyAirbyteInputError( + input_value=( + "`--sample-config` is required when `--validate-install-only`" + "is not set." + ) + ) + full_tests(connector_name, sample_config) + finally: + del os.environ["AIRBYTE_LOCAL_REGISTRY"] + temp_file.close() diff --git a/poetry.lock b/poetry.lock index 8415d8ea..042c2530 100644 --- a/poetry.lock +++ b/poetry.lock @@ -69,13 +69,13 @@ vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings [[package]] name = "airbyte-protocol-models" -version = "0.10.0" +version = "0.11.0" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.10.0-py3-none-any.whl", hash = "sha256:84447b00492193d2852e90898b56c0eeb8387a7cdb0b033860a606cee30451be"}, - {file = "airbyte_protocol_models-0.10.0.tar.gz", hash = "sha256:03835e75104ca6feb82fa86a69afa5e970a7ddf8fd6c2d4494ff4b5a64e05c07"}, + {file = "airbyte_protocol_models-0.11.0-py3-none-any.whl", hash = "sha256:2157757c1af8c13e471ab6a0304fd2f9a2a6af8cc9173937be1348a9553f7c32"}, + {file = "airbyte_protocol_models-0.11.0.tar.gz", hash = "sha256:1c7e46251b0d5a292b4aa382df24f415ac2a2a2b4719361b3c0f76368a043c23"}, ] [package.dependencies] @@ -758,13 +758,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-cloud-bigquery" -version = "3.22.0" +version = "3.23.1" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.22.0.tar.gz", hash = "sha256:957591e6f948d7cb4aa0f7a8e4e47b4617cd7f0269e28a71c37953c39b6e8a4c"}, - {file = "google_cloud_bigquery-3.22.0-py2.py3-none-any.whl", hash = "sha256:80c8e31a23b68b7d3ae5d138c9a9edff69d100ee812db73a5e63c79a13a5063d"}, + {file = "google-cloud-bigquery-3.23.1.tar.gz", hash = "sha256:4b4597f9291b42102c9667d3b4528f801d4c8f24ef2b12dd1ecb881273330955"}, + {file = "google_cloud_bigquery-3.23.1-py2.py3-none-any.whl", hash = "sha256:9fb72884fdbec9c4643cea6b7f21e1ecf3eb61d5305f87493d271dc801647a9e"}, ] [package.dependencies] @@ -787,6 +787,31 @@ opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation ( pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] +[[package]] +name = "google-cloud-bigquery-storage" +version = "2.25.0" +description = "Google Cloud Bigquery Storage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-bigquery-storage-2.25.0.tar.gz", hash = "sha256:195ffd2b180e9eaa22beab3e40a0b61ecd146ecb32667cf42269a8d7111b4d48"}, + {file = "google_cloud_bigquery_storage-2.25.0-py2.py3-none-any.whl", hash = "sha256:01109a66d1d23745d8d9c30852a0a137e8238faac1b3b02dec0d8e319ca231cb"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" +proto-plus = [ + {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[package.extras] +fastavro = ["fastavro (>=0.21.2)"] +pandas = ["importlib-metadata (>=1.0.0)", "pandas (>=0.21.1)"] +pyarrow = ["pyarrow (>=0.15.0)"] + [[package]] name = "google-cloud-core" version = "2.4.1" @@ -1028,61 +1053,61 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "grpcio" -version = "1.63.0" +version = "1.64.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.63.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2e93aca840c29d4ab5db93f94ed0a0ca899e241f2e8aec6334ab3575dc46125c"}, - {file = "grpcio-1.63.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:91b73d3f1340fefa1e1716c8c1ec9930c676d6b10a3513ab6c26004cb02d8b3f"}, - {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:b3afbd9d6827fa6f475a4f91db55e441113f6d3eb9b7ebb8fb806e5bb6d6bd0d"}, - {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f3f6883ce54a7a5f47db43289a0a4c776487912de1a0e2cc83fdaec9685cc9f"}, - {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf8dae9cc0412cb86c8de5a8f3be395c5119a370f3ce2e69c8b7d46bb9872c8d"}, - {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:08e1559fd3b3b4468486b26b0af64a3904a8dbc78d8d936af9c1cf9636eb3e8b"}, - {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5c039ef01516039fa39da8a8a43a95b64e288f79f42a17e6c2904a02a319b357"}, - {file = "grpcio-1.63.0-cp310-cp310-win32.whl", hash = "sha256:ad2ac8903b2eae071055a927ef74121ed52d69468e91d9bcbd028bd0e554be6d"}, - {file = "grpcio-1.63.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2e44f59316716532a993ca2966636df6fbe7be4ab6f099de6815570ebe4383a"}, - {file = "grpcio-1.63.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:f28f8b2db7b86c77916829d64ab21ff49a9d8289ea1564a2b2a3a8ed9ffcccd3"}, - {file = "grpcio-1.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:65bf975639a1f93bee63ca60d2e4951f1b543f498d581869922910a476ead2f5"}, - {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b5194775fec7dc3dbd6a935102bb156cd2c35efe1685b0a46c67b927c74f0cfb"}, - {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4cbb2100ee46d024c45920d16e888ee5d3cf47c66e316210bc236d5bebc42b3"}, - {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff737cf29b5b801619f10e59b581869e32f400159e8b12d7a97e7e3bdeee6a2"}, - {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd1e68776262dd44dedd7381b1a0ad09d9930ffb405f737d64f505eb7f77d6c7"}, - {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:93f45f27f516548e23e4ec3fbab21b060416007dbe768a111fc4611464cc773f"}, - {file = "grpcio-1.63.0-cp311-cp311-win32.whl", hash = "sha256:878b1d88d0137df60e6b09b74cdb73db123f9579232c8456f53e9abc4f62eb3c"}, - {file = "grpcio-1.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:756fed02dacd24e8f488f295a913f250b56b98fb793f41d5b2de6c44fb762434"}, - {file = "grpcio-1.63.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:93a46794cc96c3a674cdfb59ef9ce84d46185fe9421baf2268ccb556f8f81f57"}, - {file = "grpcio-1.63.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a7b19dfc74d0be7032ca1eda0ed545e582ee46cd65c162f9e9fc6b26ef827dc6"}, - {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8064d986d3a64ba21e498b9a376cbc5d6ab2e8ab0e288d39f266f0fca169b90d"}, - {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:219bb1848cd2c90348c79ed0a6b0ea51866bc7e72fa6e205e459fedab5770172"}, - {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2d60cd1d58817bc5985fae6168d8b5655c4981d448d0f5b6194bbcc038090d2"}, - {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e350cb096e5c67832e9b6e018cf8a0d2a53b2a958f6251615173165269a91b0"}, - {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:56cdf96ff82e3cc90dbe8bac260352993f23e8e256e063c327b6cf9c88daf7a9"}, - {file = "grpcio-1.63.0-cp312-cp312-win32.whl", hash = "sha256:3a6d1f9ea965e750db7b4ee6f9fdef5fdf135abe8a249e75d84b0a3e0c668a1b"}, - {file = "grpcio-1.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:d2497769895bb03efe3187fb1888fc20e98a5f18b3d14b606167dacda5789434"}, - {file = "grpcio-1.63.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fdf348ae69c6ff484402cfdb14e18c1b0054ac2420079d575c53a60b9b2853ae"}, - {file = "grpcio-1.63.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a3abfe0b0f6798dedd2e9e92e881d9acd0fdb62ae27dcbbfa7654a57e24060c0"}, - {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6ef0ad92873672a2a3767cb827b64741c363ebaa27e7f21659e4e31f4d750280"}, - {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b416252ac5588d9dfb8a30a191451adbf534e9ce5f56bb02cd193f12d8845b7f"}, - {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3b77eaefc74d7eb861d3ffbdf91b50a1bb1639514ebe764c47773b833fa2d91"}, - {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b005292369d9c1f80bf70c1db1c17c6c342da7576f1c689e8eee4fb0c256af85"}, - {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdcda1156dcc41e042d1e899ba1f5c2e9f3cd7625b3d6ebfa619806a4c1aadda"}, - {file = "grpcio-1.63.0-cp38-cp38-win32.whl", hash = "sha256:01799e8649f9e94ba7db1aeb3452188048b0019dc37696b0f5ce212c87c560c3"}, - {file = "grpcio-1.63.0-cp38-cp38-win_amd64.whl", hash = "sha256:6a1a3642d76f887aa4009d92f71eb37809abceb3b7b5a1eec9c554a246f20e3a"}, - {file = "grpcio-1.63.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:75f701ff645858a2b16bc8c9fc68af215a8bb2d5a9b647448129de6e85d52bce"}, - {file = "grpcio-1.63.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cacdef0348a08e475a721967f48206a2254a1b26ee7637638d9e081761a5ba86"}, - {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:0697563d1d84d6985e40ec5ec596ff41b52abb3fd91ec240e8cb44a63b895094"}, - {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6426e1fb92d006e47476d42b8f240c1d916a6d4423c5258ccc5b105e43438f61"}, - {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e48cee31bc5f5a31fb2f3b573764bd563aaa5472342860edcc7039525b53e46a"}, - {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:50344663068041b34a992c19c600236e7abb42d6ec32567916b87b4c8b8833b3"}, - {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:259e11932230d70ef24a21b9fb5bb947eb4703f57865a404054400ee92f42f5d"}, - {file = "grpcio-1.63.0-cp39-cp39-win32.whl", hash = "sha256:a44624aad77bf8ca198c55af811fd28f2b3eaf0a50ec5b57b06c034416ef2d0a"}, - {file = "grpcio-1.63.0-cp39-cp39-win_amd64.whl", hash = "sha256:166e5c460e5d7d4656ff9e63b13e1f6029b122104c1633d5f37eaea348d7356d"}, - {file = "grpcio-1.63.0.tar.gz", hash = "sha256:f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1"}, + {file = "grpcio-1.64.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:3b09c3d9de95461214a11d82cc0e6a46a6f4e1f91834b50782f932895215e5db"}, + {file = "grpcio-1.64.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:7e013428ab472892830287dd082b7d129f4d8afef49227a28223a77337555eaa"}, + {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:02cc9cc3f816d30f7993d0d408043b4a7d6a02346d251694d8ab1f78cc723e7e"}, + {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f5de082d936e0208ce8db9095821361dfa97af8767a6607ae71425ac8ace15c"}, + {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b7bf346391dffa182fba42506adf3a84f4a718a05e445b37824136047686a1"}, + {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b2cbdfba18408389a1371f8c2af1659119e1831e5ed24c240cae9e27b4abc38d"}, + {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca4f15427d2df592e0c8f3d38847e25135e4092d7f70f02452c0e90d6a02d6d"}, + {file = "grpcio-1.64.0-cp310-cp310-win32.whl", hash = "sha256:7c1f5b2298244472bcda49b599be04579f26425af0fd80d3f2eb5fd8bc84d106"}, + {file = "grpcio-1.64.0-cp310-cp310-win_amd64.whl", hash = "sha256:73f84f9e5985a532e47880b3924867de16fa1aa513fff9b26106220c253c70c5"}, + {file = "grpcio-1.64.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2a18090371d138a57714ee9bffd6c9c9cb2e02ce42c681aac093ae1e7189ed21"}, + {file = "grpcio-1.64.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:59c68df3a934a586c3473d15956d23a618b8f05b5e7a3a904d40300e9c69cbf0"}, + {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b52e1ec7185512103dd47d41cf34ea78e7a7361ba460187ddd2416b480e0938c"}, + {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d598b5d5e2c9115d7fb7e2cb5508d14286af506a75950762aa1372d60e41851"}, + {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01615bbcae6875eee8091e6b9414072f4e4b00d8b7e141f89635bdae7cf784e5"}, + {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0b2dfe6dcace264807d9123d483d4c43274e3f8c39f90ff51de538245d7a4145"}, + {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7f17572dc9acd5e6dfd3014d10c0b533e9f79cd9517fc10b0225746f4c24b58e"}, + {file = "grpcio-1.64.0-cp311-cp311-win32.whl", hash = "sha256:6ec5ed15b4ffe56e2c6bc76af45e6b591c9be0224b3fb090adfb205c9012367d"}, + {file = "grpcio-1.64.0-cp311-cp311-win_amd64.whl", hash = "sha256:597191370951b477b7a1441e1aaa5cacebeb46a3b0bd240ec3bb2f28298c7553"}, + {file = "grpcio-1.64.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:1ce4cd5a61d4532651079e7aae0fedf9a80e613eed895d5b9743e66b52d15812"}, + {file = "grpcio-1.64.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:650a8150a9b288f40d5b7c1d5400cc11724eae50bd1f501a66e1ea949173649b"}, + {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8de0399b983f8676a7ccfdd45e5b2caec74a7e3cc576c6b1eecf3b3680deda5e"}, + {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46b8b43ba6a2a8f3103f103f97996cad507bcfd72359af6516363c48793d5a7b"}, + {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a54362f03d4dcfae63be455d0a7d4c1403673498b92c6bfe22157d935b57c7a9"}, + {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1f8ea18b928e539046bb5f9c124d717fbf00cc4b2d960ae0b8468562846f5aa1"}, + {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c56c91bd2923ddb6e7ed28ebb66d15633b03e0df22206f22dfcdde08047e0a48"}, + {file = "grpcio-1.64.0-cp312-cp312-win32.whl", hash = "sha256:874c741c8a66f0834f653a69e7e64b4e67fcd4a8d40296919b93bab2ccc780ba"}, + {file = "grpcio-1.64.0-cp312-cp312-win_amd64.whl", hash = "sha256:0da1d921f8e4bcee307aeef6c7095eb26e617c471f8cb1c454fd389c5c296d1e"}, + {file = "grpcio-1.64.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c46fb6bfca17bfc49f011eb53416e61472fa96caa0979b4329176bdd38cbbf2a"}, + {file = "grpcio-1.64.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3d2004e85cf5213995d09408501f82c8534700d2babeb81dfdba2a3bff0bb396"}, + {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6d5541eb460d73a07418524fb64dcfe0adfbcd32e2dac0f8f90ce5b9dd6c046c"}, + {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f279ad72dd7d64412e10f2443f9f34872a938c67387863c4cd2fb837f53e7d2"}, + {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85fda90b81da25993aa47fae66cae747b921f8f6777550895fb62375b776a231"}, + {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a053584079b793a54bece4a7d1d1b5c0645bdbee729215cd433703dc2532f72b"}, + {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:579dd9fb11bc73f0de061cab5f8b2def21480fd99eb3743ed041ad6a1913ee2f"}, + {file = "grpcio-1.64.0-cp38-cp38-win32.whl", hash = "sha256:23b6887bb21d77649d022fa1859e05853fdc2e60682fd86c3db652a555a282e0"}, + {file = "grpcio-1.64.0-cp38-cp38-win_amd64.whl", hash = "sha256:753cb58683ba0c545306f4e17dabf468d29cb6f6b11832e1e432160bb3f8403c"}, + {file = "grpcio-1.64.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:2186d76a7e383e1466e0ea2b0febc343ffeae13928c63c6ec6826533c2d69590"}, + {file = "grpcio-1.64.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0f30596cdcbed3c98024fb4f1d91745146385b3f9fd10c9f2270cbfe2ed7ed91"}, + {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:d9171f025a196f5bcfec7e8e7ffb7c3535f7d60aecd3503f9e250296c7cfc150"}, + {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf4c8daed18ae2be2f1fc7d613a76ee2a2e28fdf2412d5c128be23144d28283d"}, + {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3550493ac1d23198d46dc9c9b24b411cef613798dc31160c7138568ec26bc9b4"}, + {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3161a8f8bb38077a6470508c1a7301cd54301c53b8a34bb83e3c9764874ecabd"}, + {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8fabe2cc57a369638ab1ad8e6043721014fdf9a13baa7c0e35995d3a4a7618"}, + {file = "grpcio-1.64.0-cp39-cp39-win32.whl", hash = "sha256:31890b24d47b62cc27da49a462efe3d02f3c120edb0e6c46dcc0025506acf004"}, + {file = "grpcio-1.64.0-cp39-cp39-win_amd64.whl", hash = "sha256:5a56797dea8c02e7d3a85dfea879f286175cf4d14fbd9ab3ef2477277b927baa"}, + {file = "grpcio-1.64.0.tar.gz", hash = "sha256:257baf07f53a571c215eebe9679c3058a313fd1d1f7c4eede5a8660108c52d9c"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.63.0)"] +protobuf = ["grpcio-tools (>=1.64.0)"] [[package]] name = "grpcio-status" @@ -1245,13 +1270,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.58" +version = "0.1.60" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.58-py3-none-any.whl", hash = "sha256:1148cc836ec99d1b2f37cd2fa3014fcac213bb6bad798a2b21bb9111c18c9768"}, - {file = "langsmith-0.1.58.tar.gz", hash = "sha256:a5060933c1fb3006b498ec849677993329d7e6138bdc2ec044068ab806e09c39"}, + {file = "langsmith-0.1.60-py3-none-any.whl", hash = "sha256:3c3520ea473de0a984237b3e9d638fdf23ef3acc5aec89a42e693225e72d6120"}, + {file = "langsmith-0.1.60.tar.gz", hash = "sha256:6a145b5454437f9e0f81525f23c4dcdbb8c07b1c91553b8f697456c418d6a599"}, ] [package.dependencies] @@ -1673,13 +1698,13 @@ files = [ [[package]] name = "pdoc" -version = "14.4.0" +version = "14.5.0" description = "API Documentation for Python Projects" optional = false python-versions = ">=3.8" files = [ - {file = "pdoc-14.4.0-py3-none-any.whl", hash = "sha256:6ea4fe07620b1f7601e2708a307a257636ec206e20b5611640b30f2e3cab47d6"}, - {file = "pdoc-14.4.0.tar.gz", hash = "sha256:c92edc425429ccbe287ace2a027953c24f13de53eab484c1a6d31ca72dd2fda9"}, + {file = "pdoc-14.5.0-py3-none-any.whl", hash = "sha256:9a8a84e19662610c0620fbe9f2e4174e3b090f8b601ed46348786ebb7517c508"}, + {file = "pdoc-14.5.0.tar.gz", hash = "sha256:79f534dc8a6494638dd6056b78e17a654df7ed34cc92646553ce3a7ba5a4fa4a"}, ] [package.dependencies] @@ -2086,13 +2111,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.4" +version = "8.2.1" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, + {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, ] [package.dependencies] @@ -2100,30 +2125,30 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-docker" -version = "2.2.0" +version = "3.1.1" description = "Simple pytest fixtures for Docker and Docker Compose based tests" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "pytest-docker-2.2.0.tar.gz", hash = "sha256:b083fd2ae69212369390033c22228d3263555a5f3b4bef87b74160e07218f377"}, - {file = "pytest_docker-2.2.0-py3-none-any.whl", hash = "sha256:8ee9c9742d58ac079c81c03635bb830881f7f4d529f0f53f4ba2c89ffc9c7137"}, + {file = "pytest-docker-3.1.1.tar.gz", hash = "sha256:2371524804a752aaa766c79b9eee8e634534afddb82597f3b573da7c5d6ffb5f"}, + {file = "pytest_docker-3.1.1-py3-none-any.whl", hash = "sha256:fd0d48d6feac41f62acbc758319215ec9bb805c2309622afb07c27fa5c5ae362"}, ] [package.dependencies] attrs = ">=19.2.0" -pytest = ">=4.0,<8.0" +pytest = ">=4.0,<9.0" [package.extras] docker-compose-v1 = ["docker-compose (>=1.27.3,<2.0)"] -tests = ["pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)"] +tests = ["mypy (>=0.500,<2.000)", "pytest-mypy (>=0.10,<1.0)", "pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)", "types-requests (>=2.31,<3.0)", "types-setuptools (>=69.0,<70.0)"] [[package]] name = "pytest-mock" @@ -2579,19 +2604,18 @@ files = [ [[package]] name = "setuptools" -version = "69.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, - {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -2606,37 +2630,37 @@ files = [ [[package]] name = "snowflake-connector-python" -version = "3.10.0" +version = "3.10.1" description = "Snowflake Connector for Python" optional = false python-versions = ">=3.8" files = [ - {file = "snowflake_connector_python-3.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e2afca4bca70016519d1a7317c498f1d9c56140bf3e40ea40bddcc95fe827ca"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:d19bde29f89b226eb22af4c83134ecb5c229da1d5e960a01b8f495df78dcdc36"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bfe013ed97b4dd2e191fd6770a14030d29dd0108817d6ce76b9773250dd2d560"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0917c9f9382d830907e1a18ee1208537b203618700a9c671c2a20167b30f574"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:7e828bc99240433e6552ac4cc4e37f223ae5c51c7880458ddb281668503c7491"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0d3d06d758455c50b998eabc1fd972a1f67faa5c85ef250fd5986f5a41aab0b"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:4602cb19b204bb03e03d65c6d5328467c9efc0fec53ca56768c3747c8dc8a70f"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb1a04b496bbd3e1e2e926df82b2369887b2eea958f535fb934c240bfbabf6c5"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c889f9f60f915d657e0a0ad2e6cc52cdcafd9bcbfa95a095aadfd8bcae62b819"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e441484216ed416a6ed338133e23bd991ac4ba2e46531f4d330f61568c49314"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bb4aced19053c67513cecc92311fa9d3b507b2277698c8e987d404f6f3a49fb2"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:858315a2feff86213b079c6293ad8d850a778044c664686802ead8bb1337e1bc"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:adf16e1ca9f46d3bdf68e955ffa42075ebdb251e3b13b59003d04e4fea7d579a"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4c5c2a08b39086a5348502652ad4fdf24871d7ab30fd59f6b7b57249158468c"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:05011286f42c52eb3e5a6db59ee3eaf79f3039f3a19d7ffac6f4ee143779c637"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:569301289ada5b0d72d0bd8432b7ca180220335faa6d9a0f7185f60891db6f2c"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:4e5641c70a12da9804b74f350b8cbbdffdc7aca5069b096755abd2a1fdcf5d1b"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ff767a1b8c48431549ac28884f8bd9647e63a23f470b05f6ab8d143c4b1475"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e52bbc1e2e7bda956525b4229d7f87579f8cabd7d5506b12aa754c4bcdc8c8d7"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:280a8dcca0249e864419564e38764c08f8841900d9872fec2f2855fda494b29f"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:67bf570230b0cf818e6766c17245c7355a1f5ea27778e54ab8d09e5bb3536ad9"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:aa1e26f9c571d2c4206da5c978c1b345ffd798d3db1f9ae91985e6243c6bf94b"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73e9baa531d5156a03bfe5af462cf6193ec2a01cbb575edf7a2dd3b2a35254c7"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e03361c4749e4d65bf0d223fdea1c2d7a33af53b74e873929a6085d150aff17e"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:e8cddd4357e70ab55d7aeeed144cbbeb1ff658b563d7d8d307afc06178a367ec"}, - {file = "snowflake_connector_python-3.10.0.tar.gz", hash = "sha256:7c7438e958753bd1174b73581d77c92b0b47a86c38d8ea0ba1ea23c442eb8e75"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69ccf052924995a77d6bdcf3de1b66ed90bf79ba150e02f614a5fc9a71812139"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:96bf0cc66e06ba9583cae412529ab2e0e38379a2f444c2978f25e7a033df5aad"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d011ee7f4d1461a90abe42d406e942c5089f34bcf151e81a89c5d14f195b474"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3c00a7109995df4379b3464e04ec39314ef82d2f64898323b0a5621f26af36c"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:b22131059a6aabd3bcf6e3ed5eae52b8fcc5e4900a89c209c837107cb30b5ab2"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82f26632b2d0a9003a2c1ca1987294e73383a8a68bf260443706c506314a4b88"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:c1b76618f4607985f875dab264f3d281db2dcbf3db726e1a29b86aa22681dcf6"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d34fb7af5a14bd373928193b231647123b42df50f2f2069af0714c6bd1c99726"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424c7cca46daafec34a77a664f925b771fe38a31035373db61fc4134994a5501"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:b1a8ae9df534891e2261cc8bac272bc8cc76b4e54b8b182e7d7a136cedb6e2e1"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cdd5bb449e0e6c493a6602ee2098ae71d77852acd39f28bf75ccf7ef4fcaca5"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:42e6d2a2327a8f25a6c68b4308a44d4fc442be0b3714805aadb0236fb17cc610"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86a676643953e0a460872dd52c67263a5ca29bea6e7020492b43ab5d99ebd48d"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a62150cb59b56b299e5715188bb244b233d0f16d56a2d26c35a7e04c3c40bbc"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:8a4da738539a8c16baf7798e2f6382e3f29adf27988047de0445295b13eb9992"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5ee40801c4b1e3b8acd9c003e044d0275f7a49d56f96618fe164eb1245e6d9"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:2e59ca84acaa5309ced48ef6c8e41532f2c4665c084800737f406cbad2caee0d"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:886daab478fc704c1186cbed36f563cbe9623bcbe5dc9fe33e77cea17a40d9eb"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:480e9f586eaeec657ebe329ed7c6a5cde87f1331f5b2d0a672fb330cff6403ed"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:7253a8f0e33d99f98f616b652a156a359f9c458b943639ae93e219718254ebaa"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bbe83707cf356285df2f3cdaf542d5416dc93d833c1c191ec896ba0c01f7c302"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:0edd33f090c40b6e60d524f4107b4c7a0353312766bf3a0a1c5a4165a87adf7f"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48bbc5c7b33a1e296732a97799448af4cf4b8a71bd5a1d0def667a418fe7f1e"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73a98c72a72119fd6ac3ac141ace55572727fd2ba4239e0abf82d4f7016630f1"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:ea66d9a0a212452974f2a591a9ac0657e9982c0e9fa52e277372da4959d4ad21"}, + {file = "snowflake_connector_python-3.10.1.tar.gz", hash = "sha256:b9214da76ce72fff8eb60066fde6b9ba7f58a055d68ffe1e7a9b1034f57a34b4"}, ] [package.dependencies] @@ -3068,4 +3092,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "8c72f18f4c6b45646c9039019abb491f552fbc47101dbe3f5b6d97ec0699336a" +content-hash = "6e3d7eeb9e986a14dd4c9936f6710e76973c2bd6e83e861ba7ab6209a4910a3b" diff --git a/pyproject.toml b/pyproject.toml index 0271d46e..d78de9f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ psycopg2-binary = "^2.9.9" pydantic = "<=2.0" python-dotenv = "^1.0.1" python-ulid = "^2.2.0" -requests = "^2.31.0" +requests = "<=2.31.0" # 3.32.0 breaks docker tests rich = "^13.7.0" snowflake-connector-python = "^3.10.0" snowflake-sqlalchemy = "^1.5.1" @@ -43,6 +43,7 @@ ulid = "^1.1" # TODO: Remove this arbitrary python constraint once `sqlalchemy-bigquery` has done so. sqlalchemy-bigquery = { version = "1.9.0", python = "<3.13" } airbyte-api = "^0.49.2" +google-cloud-bigquery-storage = "^2.25.0" [tool.poetry.group.dev.dependencies] docker = "^7.0.0" @@ -50,8 +51,8 @@ faker = "^21.0.0" mypy = "^1.7.1" pandas-stubs = "^2.1.4.231218" pdoc = "^14.3.0" -pytest = "^7.4.3" -pytest-docker = "^2.0.1" +pytest = "^8.2.0" +pytest-docker = "^3.1.1" pytest-mypy = "^0.10.3" ruff = "^0.4.1" types-jsonschema = "^4.20.0.0" @@ -81,6 +82,18 @@ markers = [ "requires_creds: marks a test as requiring credentials (skip when secrets unavailable)", "linting: marks a test as a linting test", ] +filterwarnings = [ + # Treat python warnings as errors in pytest + "error", + # TODO: Remove these ignores once we are able to fix the warnings: + "ignore::sqlalchemy.exc.MovedIn20Warning", + # Ignore these, specifically on Windows because of file cleanup and locking issues: + "ignore:unclosed file:ResourceWarning", + "ignore::pytest.PytestUnraisableExceptionWarning", + # Order matters here. This must be last because it will fail if we + # hit another warning during the import. + "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", +] [tool.ruff] target-version = "py39" diff --git a/tests/conftest.py b/tests/conftest.py index ac14d7d4..0e176b24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,9 @@ import shutil import socket import subprocess +import sys import time +import warnings from pathlib import Path import docker @@ -33,6 +35,20 @@ LOCAL_TEST_REGISTRY_URL = "./tests/integration_tests/fixtures/registry.json" +def pytest_configure(config): + if sys.platform.startswith("win"): + # Ignore specific warnings only on Windows + warnings.filterwarnings( + "ignore", + category=pytest.PytestUnraisableExceptionWarning, + ) + warnings.filterwarnings( + "ignore", + category=ResourceWarning, + message="unclosed file", + ) + + def pytest_collection_modifyitems(items: list[Item]) -> None: """Override default pytest behavior, sorting our tests in a sensible execution order. diff --git a/tests/docs_tests/test_docs_checked_in.py b/tests/docs_tests/test_docs_checked_in.py index 868b488f..be2189a1 100644 --- a/tests/docs_tests/test_docs_checked_in.py +++ b/tests/docs_tests/test_docs_checked_in.py @@ -3,14 +3,17 @@ import os import docs.generate as generate +import pytest -def test_docs_checked_in(): +@pytest.mark.filterwarnings("ignore") +def test_docs_generation(): """ - Docs need to be generated via `poetry run generate-docs` and checked in to the repo. + Docs need to be able to be generated via `poetry run generate-docs`. - This test runs the docs generation and compares the output with the checked in docs. - It will fail if there are any differences. + This test runs the docs generation and ensures that it can complete successfully. + + Generation often produces warnings that are not relevant, so we suppress warnings in this test. """ generate.run() diff --git a/tests/integration_tests/cloud/test_cloud_sql_reads.py b/tests/integration_tests/cloud/test_cloud_sql_reads.py index 6cb48afa..25e7b074 100644 --- a/tests/integration_tests/cloud/test_cloud_sql_reads.py +++ b/tests/integration_tests/cloud/test_cloud_sql_reads.py @@ -5,13 +5,12 @@ from contextlib import suppress +import airbyte as ab import pandas as pd import pytest -from sqlalchemy.engine.base import Engine - -import airbyte as ab from airbyte import cloud from airbyte.cloud.sync_results import SyncResult +from sqlalchemy.engine.base import Engine @pytest.fixture @@ -109,7 +108,7 @@ def test_read_from_deployed_connection( cache = sync_result.get_sql_cache() sqlalchemy_url = cache.get_sql_alchemy_url() engine: Engine = sync_result.get_sql_engine() - # assert sync_result.stream_names == ["users", "products", "purchases"] + assert "users" in sync_result.stream_names dataset: ab.CachedDataset = sync_result.get_dataset(stream_name="users") assert dataset.stream_name == "users" diff --git a/tests/integration_tests/test_all_cache_types.py b/tests/integration_tests/test_all_cache_types.py index e7c4087f..f28f87cd 100644 --- a/tests/integration_tests/test_all_cache_types.py +++ b/tests/integration_tests/test_all_cache_types.py @@ -54,11 +54,8 @@ def source_faker_seed_a() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users"], ) - source.check() - source.select_streams([ - "users", - ]) return source @@ -74,11 +71,8 @@ def source_faker_seed_b() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users"], ) - source.check() - source.select_streams([ - "users", - ]) return source @@ -94,7 +88,6 @@ def source_pokeapi() -> ab.Source: install_if_missing=False, # Should already be on PATH streams="*", ) - source.check() return source diff --git a/tests/integration_tests/test_docker_executable.py b/tests/integration_tests/test_docker_executable.py index 4d5998fe..47f18a6e 100644 --- a/tests/integration_tests/test_docker_executable.py +++ b/tests/integration_tests/test_docker_executable.py @@ -42,13 +42,8 @@ def source_docker_faker_seed_a() -> ab.Source: "seed": SEED_A, "parallelism": 16, # Otherwise defaults to 4. }, + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "users", - "products", - "purchases", - ]) return source @@ -63,13 +58,8 @@ def source_docker_faker_seed_b() -> ab.Source: "seed": SEED_B, "parallelism": 16, # Otherwise defaults to 4. }, + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "products", - "purchases", - "users", - ]) return source diff --git a/tests/integration_tests/test_duckdb_cache.py b/tests/integration_tests/test_duckdb_cache.py index 45ce7fe8..5022ccc1 100644 --- a/tests/integration_tests/test_duckdb_cache.py +++ b/tests/integration_tests/test_duckdb_cache.py @@ -8,15 +8,13 @@ from __future__ import annotations -from collections.abc import Generator import os -from pathlib import Path import sys - -import pytest - +from collections.abc import Generator +from pathlib import Path import airbyte as ab +import pytest from airbyte._executor import _get_bin_dir from airbyte.caches.duckdb import DuckDBCache from airbyte.caches.util import new_local_cache @@ -44,9 +42,8 @@ def add_venv_bin_to_path(monkeypatch): monkeypatch.setenv("PATH", new_path) -@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. -def source_faker() -> ab.Source: - """Fixture to return a source-faker connector instance.""" +def setup_source_faker() -> ab.Source: + """Test the source-faker setup.""" source = ab.get_source( "source-faker", local_executable="source-faker", @@ -66,6 +63,17 @@ def source_faker() -> ab.Source: return source +@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. +def source_faker() -> ab.Source: + """Fixture to return a source-faker connector instance.""" + return setup_source_faker() + + +def test_setup_source_faker() -> None: + """Test that fixture logic works as expected.""" + source = setup_source_faker() + + @pytest.fixture(scope="function") def duckdb_cache() -> Generator[DuckDBCache, None, None]: """Fixture to return a fresh cache.""" diff --git a/tests/integration_tests/test_source_faker_integration.py b/tests/integration_tests/test_source_faker_integration.py index 97e30053..6de903c2 100644 --- a/tests/integration_tests/test_source_faker_integration.py +++ b/tests/integration_tests/test_source_faker_integration.py @@ -12,6 +12,7 @@ import shutil import sys import tempfile +import warnings from collections.abc import Generator from pathlib import Path @@ -27,6 +28,7 @@ from airbyte.caches.postgres import PostgresCache from airbyte.caches.util import new_local_cache from airbyte.strategies import WriteStrategy +from duckdb_engine import DuckDBEngineWarning # Product count is always the same, regardless of faker scale. NUM_PRODUCTS = 100 @@ -65,13 +67,8 @@ def source_faker_seed_a() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "users", - "products", - "purchases", - ]) return source @@ -87,19 +84,22 @@ def source_faker_seed_b() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "products", - "purchases", - "users", - ]) return source @pytest.fixture(scope="function") def duckdb_cache() -> Generator[DuckDBCache, None, None]: """Fixture to return a fresh cache.""" + # Suppress warnings from DuckDB about reflection on indices. + # https://github.com/Mause/duckdb_engine/issues/905 + warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, + ) + cache: DuckDBCache = new_local_cache() yield cache # TODO: Delete cache DB file after test is complete.