From 013bd82199d920582401dee3a38087004da20c94 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 17 May 2024 11:09:24 -0700 Subject: [PATCH 01/30] chore: configure warnings as errors in pytest --- poetry.lock | 59 ++++++++++++++++++++------------------------------ pyproject.toml | 10 +++++++-- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8415d8ea..8210feec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-api" @@ -69,13 +69,13 @@ vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings [[package]] name = "airbyte-protocol-models" -version = "0.10.0" +version = "0.11.0" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.10.0-py3-none-any.whl", hash = "sha256:84447b00492193d2852e90898b56c0eeb8387a7cdb0b033860a606cee30451be"}, - {file = "airbyte_protocol_models-0.10.0.tar.gz", hash = "sha256:03835e75104ca6feb82fa86a69afa5e970a7ddf8fd6c2d4494ff4b5a64e05c07"}, + {file = "airbyte_protocol_models-0.11.0-py3-none-any.whl", hash = "sha256:2157757c1af8c13e471ab6a0304fd2f9a2a6af8cc9173937be1348a9553f7c32"}, + {file = "airbyte_protocol_models-0.11.0.tar.gz", hash = "sha256:1c7e46251b0d5a292b4aa382df24f415ac2a2a2b4719361b3c0f76368a043c23"}, ] [package.dependencies] @@ -758,13 +758,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-cloud-bigquery" -version = "3.22.0" +version = "3.23.0" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.22.0.tar.gz", hash = "sha256:957591e6f948d7cb4aa0f7a8e4e47b4617cd7f0269e28a71c37953c39b6e8a4c"}, - {file = "google_cloud_bigquery-3.22.0-py2.py3-none-any.whl", hash = "sha256:80c8e31a23b68b7d3ae5d138c9a9edff69d100ee812db73a5e63c79a13a5063d"}, + {file = "google-cloud-bigquery-3.23.0.tar.gz", hash = "sha256:7ecdb207727d513b1bce1f213dbb926ed2e1d4f0122778de00f0e56d19d47a01"}, + {file = "google_cloud_bigquery-3.23.0-py2.py3-none-any.whl", hash = "sha256:dc0a4a47ab541a34aa1dc1f48539d88c091adc0637da7744d7fab6f3bc8886d5"}, ] [package.dependencies] @@ -1245,13 +1245,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.58" +version = "0.1.59" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.58-py3-none-any.whl", hash = "sha256:1148cc836ec99d1b2f37cd2fa3014fcac213bb6bad798a2b21bb9111c18c9768"}, - {file = "langsmith-0.1.58.tar.gz", hash = "sha256:a5060933c1fb3006b498ec849677993329d7e6138bdc2ec044068ab806e09c39"}, + {file = "langsmith-0.1.59-py3-none-any.whl", hash = "sha256:445e3bc1d3baa1e5340cd979907a19483b9763a2ed37b863a01113d406f69345"}, + {file = "langsmith-0.1.59.tar.gz", hash = "sha256:e748a89f4dd6aa441349143e49e546c03b5dfb43376a25bfef6a5ca792fe1437"}, ] [package.dependencies] @@ -2086,13 +2086,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.4" +version = "8.2.0" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"}, + {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"}, ] [package.dependencies] @@ -2100,30 +2100,30 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-docker" -version = "2.2.0" +version = "3.1.1" description = "Simple pytest fixtures for Docker and Docker Compose based tests" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "pytest-docker-2.2.0.tar.gz", hash = "sha256:b083fd2ae69212369390033c22228d3263555a5f3b4bef87b74160e07218f377"}, - {file = "pytest_docker-2.2.0-py3-none-any.whl", hash = "sha256:8ee9c9742d58ac079c81c03635bb830881f7f4d529f0f53f4ba2c89ffc9c7137"}, + {file = "pytest-docker-3.1.1.tar.gz", hash = "sha256:2371524804a752aaa766c79b9eee8e634534afddb82597f3b573da7c5d6ffb5f"}, + {file = "pytest_docker-3.1.1-py3-none-any.whl", hash = "sha256:fd0d48d6feac41f62acbc758319215ec9bb805c2309622afb07c27fa5c5ae362"}, ] [package.dependencies] attrs = ">=19.2.0" -pytest = ">=4.0,<8.0" +pytest = ">=4.0,<9.0" [package.extras] docker-compose-v1 = ["docker-compose (>=1.27.3,<2.0)"] -tests = ["pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)"] +tests = ["mypy (>=0.500,<2.000)", "pytest-mypy (>=0.10,<1.0)", "pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)", "types-requests (>=2.31,<3.0)", "types-setuptools (>=69.0,<70.0)"] [[package]] name = "pytest-mock" @@ -2278,7 +2278,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2286,16 +2285,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2312,7 +2303,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2320,7 +2310,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -3068,4 +3057,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "8c72f18f4c6b45646c9039019abb491f552fbc47101dbe3f5b6d97ec0699336a" +content-hash = "d0219e61a40e56f4bb4ff470d43d4d35510cf0222de19bc51d8b64a6c9847ebf" diff --git a/pyproject.toml b/pyproject.toml index 0271d46e..0658fe51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ ulid = "^1.1" # TODO: Remove this arbitrary python constraint once `sqlalchemy-bigquery` has done so. sqlalchemy-bigquery = { version = "1.9.0", python = "<3.13" } airbyte-api = "^0.49.2" +colorama = "^0.4.6" [tool.poetry.group.dev.dependencies] docker = "^7.0.0" @@ -50,8 +51,8 @@ faker = "^21.0.0" mypy = "^1.7.1" pandas-stubs = "^2.1.4.231218" pdoc = "^14.3.0" -pytest = "^7.4.3" -pytest-docker = "^2.0.1" +pytest = "^8.2.0" +pytest-docker = "^3.1.1" pytest-mypy = "^0.10.3" ruff = "^0.4.1" types-jsonschema = "^4.20.0.0" @@ -81,6 +82,11 @@ markers = [ "requires_creds: marks a test as requiring credentials (skip when secrets unavailable)", "linting: marks a test as a linting test", ] +# Treat python warnings as errors in pytest +filterwarnings = [ + "error", + "ignore::sqlalchemy.exc.MovedIn20Warning", +] [tool.ruff] target-version = "py39" From f2804d390982798c3d8d89f9c2a972065d87373e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 20 May 2024 14:24:12 -0700 Subject: [PATCH 02/30] feat: add new custom warning for `AirbyteExperimentalFeatureWarning` --- airbyte/cloud/experimental.py | 3 ++- airbyte/exceptions.py | 7 +++++++ pyproject.toml | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/airbyte/cloud/experimental.py b/airbyte/cloud/experimental.py index fbc3ace4..91bd6223 100644 --- a/airbyte/cloud/experimental.py +++ b/airbyte/cloud/experimental.py @@ -27,6 +27,7 @@ import warnings +from airbyte import exceptions as exc from airbyte.cloud.connections import CloudConnection as Stable_CloudConnection from airbyte.cloud.workspaces import CloudWorkspace as Stable_CloudWorkspace @@ -35,7 +36,7 @@ # explicitly imports it. warnings.warn( message="The `airbyte.cloud.experimental` module is experimental and may change in the future.", - category=FutureWarning, + category=exc.AirbyteExperimentalFeatureWarning, stacklevel=2, ) diff --git a/airbyte/exceptions.py b/airbyte/exceptions.py index b3b4f367..cd084446 100644 --- a/airbyte/exceptions.py +++ b/airbyte/exceptions.py @@ -397,3 +397,10 @@ class AirbyteMultipleResourcesError(AirbyteError): resource_type: str | None = None resource_name_or_id: str | None = None + + +# Custom Warnings + + +class AirbyteExperimentalFeatureWarning(FutureWarning): + """Warning whenever using experimental features in PyAirbyte.""" diff --git a/pyproject.toml b/pyproject.toml index 0658fe51..4e7a9162 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ markers = [ filterwarnings = [ "error", "ignore::sqlalchemy.exc.MovedIn20Warning", + "ignore::airbyte.exc.AirbyteExperimentalFeatureWarning", ] [tool.ruff] From e0731daee1d44e9c50b4f2a62c646242ca196d80 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 20 May 2024 14:26:16 -0700 Subject: [PATCH 03/30] fix module name --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4e7a9162..318b89ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ markers = [ filterwarnings = [ "error", "ignore::sqlalchemy.exc.MovedIn20Warning", - "ignore::airbyte.exc.AirbyteExperimentalFeatureWarning", + "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", ] [tool.ruff] From d14767a5eee33324b37de4a3688a7edb8096f057 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 20 May 2024 14:39:49 -0700 Subject: [PATCH 04/30] suppress DuckDB engine error in pytest --- airbyte/_processors/sql/duckdb.py | 2 ++ airbyte/_processors/sql/motherduck.py | 2 ++ airbyte/caches/duckdb.py | 2 ++ .../integration_tests/test_source_faker_integration.py | 10 ++++++++++ 4 files changed, 16 insertions(+) diff --git a/airbyte/_processors/sql/duckdb.py b/airbyte/_processors/sql/duckdb.py index 2257102d..603ac324 100644 --- a/airbyte/_processors/sql/duckdb.py +++ b/airbyte/_processors/sql/duckdb.py @@ -8,6 +8,7 @@ from textwrap import dedent, indent from typing import TYPE_CHECKING, Union +from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field from typing_extensions import Literal @@ -27,6 +28,7 @@ warnings.filterwarnings( "ignore", message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, ) diff --git a/airbyte/_processors/sql/motherduck.py b/airbyte/_processors/sql/motherduck.py index 37e79352..edece91e 100644 --- a/airbyte/_processors/sql/motherduck.py +++ b/airbyte/_processors/sql/motherduck.py @@ -6,6 +6,7 @@ import warnings from typing import TYPE_CHECKING +from duckdb_engine import DuckDBEngineWarning from overrides import overrides from airbyte._processors.file import JsonlWriter @@ -21,6 +22,7 @@ warnings.filterwarnings( "ignore", message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, ) diff --git a/airbyte/caches/duckdb.py b/airbyte/caches/duckdb.py index 12eb4225..47b56020 100644 --- a/airbyte/caches/duckdb.py +++ b/airbyte/caches/duckdb.py @@ -18,6 +18,7 @@ import warnings +from duckdb_engine import DuckDBEngineWarning from pydantic import PrivateAttr from airbyte._processors.sql.duckdb import DuckDBConfig, DuckDBSqlProcessor @@ -29,6 +30,7 @@ warnings.filterwarnings( "ignore", message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, ) diff --git a/tests/integration_tests/test_source_faker_integration.py b/tests/integration_tests/test_source_faker_integration.py index 97e30053..f31e65d2 100644 --- a/tests/integration_tests/test_source_faker_integration.py +++ b/tests/integration_tests/test_source_faker_integration.py @@ -12,6 +12,7 @@ import shutil import sys import tempfile +import warnings from collections.abc import Generator from pathlib import Path @@ -27,6 +28,7 @@ from airbyte.caches.postgres import PostgresCache from airbyte.caches.util import new_local_cache from airbyte.strategies import WriteStrategy +from duckdb_engine import DuckDBEngineWarning # Product count is always the same, regardless of faker scale. NUM_PRODUCTS = 100 @@ -100,6 +102,14 @@ def source_faker_seed_b() -> ab.Source: @pytest.fixture(scope="function") def duckdb_cache() -> Generator[DuckDBCache, None, None]: """Fixture to return a fresh cache.""" + # Suppress warnings from DuckDB about reflection on indices. + # https://github.com/Mause/duckdb_engine/issues/905 + warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, + ) + cache: DuckDBCache = new_local_cache() yield cache # TODO: Delete cache DB file after test is complete. From 09487121ea7eb6d9d609dbbc056258fe288098ea Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 20 May 2024 14:41:07 -0700 Subject: [PATCH 05/30] add missing library 'google-cloud-bigquery-storage' --- poetry.lock | 27 ++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8210feec..eda07897 100644 --- a/poetry.lock +++ b/poetry.lock @@ -787,6 +787,31 @@ opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation ( pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] +[[package]] +name = "google-cloud-bigquery-storage" +version = "2.25.0" +description = "Google Cloud Bigquery Storage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-bigquery-storage-2.25.0.tar.gz", hash = "sha256:195ffd2b180e9eaa22beab3e40a0b61ecd146ecb32667cf42269a8d7111b4d48"}, + {file = "google_cloud_bigquery_storage-2.25.0-py2.py3-none-any.whl", hash = "sha256:01109a66d1d23745d8d9c30852a0a137e8238faac1b3b02dec0d8e319ca231cb"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" +proto-plus = [ + {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[package.extras] +fastavro = ["fastavro (>=0.21.2)"] +pandas = ["importlib-metadata (>=1.0.0)", "pandas (>=0.21.1)"] +pyarrow = ["pyarrow (>=0.15.0)"] + [[package]] name = "google-cloud-core" version = "2.4.1" @@ -3057,4 +3082,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "d0219e61a40e56f4bb4ff470d43d4d35510cf0222de19bc51d8b64a6c9847ebf" +content-hash = "331322a76c543139350ddbb3dd716134527680ad10add1fe495bb6a596b84e6a" diff --git a/pyproject.toml b/pyproject.toml index 318b89ef..72d4111a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ ulid = "^1.1" sqlalchemy-bigquery = { version = "1.9.0", python = "<3.13" } airbyte-api = "^0.49.2" colorama = "^0.4.6" +google-cloud-bigquery-storage = "^2.25.0" [tool.poetry.group.dev.dependencies] docker = "^7.0.0" From 6b1cdd99cb1238a231c667eb1eabceb7b814f9b8 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 11:30:54 -0700 Subject: [PATCH 06/30] suppress duckdb warning during sql alchemy initialization --- airbyte/_processors/sql/duckdb.py | 16 +++++++--------- airbyte/caches/motherduck.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/airbyte/_processors/sql/duckdb.py b/airbyte/_processors/sql/duckdb.py index 603ac324..fc6a8259 100644 --- a/airbyte/_processors/sql/duckdb.py +++ b/airbyte/_processors/sql/duckdb.py @@ -23,14 +23,6 @@ from sqlalchemy.engine import Engine -# Suppress warnings from DuckDB about reflection on indices. -# https://github.com/Mause/duckdb_engine/issues/905 -warnings.filterwarnings( - "ignore", - message="duckdb-engine doesn't yet support reflection on indices", - category=DuckDBEngineWarning, -) - # @dataclass class DuckDBConfig(SqlConfig): @@ -49,7 +41,13 @@ class DuckDBConfig(SqlConfig): @overrides def get_sql_alchemy_url(self) -> SecretString: """Return the SQLAlchemy URL to use.""" - # return f"duckdb:///{self.db_path}?schema={self.schema_name}" + # Suppress warnings from DuckDB about reflection on indices. + # https://github.com/Mause/duckdb_engine/issues/905 + warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, + ) return SecretString(f"duckdb:///{self.db_path!s}") @overrides diff --git a/airbyte/caches/motherduck.py b/airbyte/caches/motherduck.py index f6dfd2cb..62a140e5 100644 --- a/airbyte/caches/motherduck.py +++ b/airbyte/caches/motherduck.py @@ -16,6 +16,9 @@ from __future__ import annotations +import warnings + +from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field, PrivateAttr @@ -35,6 +38,14 @@ class MotherDuckConfig(DuckDBConfig): @overrides def get_sql_alchemy_url(self) -> SecretString: """Return the SQLAlchemy URL to use.""" + # Suppress warnings from DuckDB about reflection on indices. + # https://github.com/Mause/duckdb_engine/issues/905 + warnings.filterwarnings( + "ignore", + message="duckdb-engine doesn't yet support reflection on indices", + category=DuckDBEngineWarning, + ) + return SecretString( f"duckdb:///md:{self.database}?motherduck_token={self.api_key}" # f"&schema={self.schema_name}" # TODO: Debug why this doesn't work From 89c8907706a509bcfbb6101b2c27bf37094ca79c Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 12:08:21 -0700 Subject: [PATCH 07/30] improve temp file handling and cleanup --- airbyte/_util/temp_files.py | 15 ++++++++++++++- airbyte/validate.py | 29 +++++++++++++++++------------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index 845d6742..d36bcab2 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -4,7 +4,9 @@ from __future__ import annotations import json +import os import tempfile +import warnings from contextlib import contextmanager, suppress from pathlib import Path from typing import TYPE_CHECKING, Any @@ -20,18 +22,29 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, temp_files: list[Any] = [] try: for content in files_contents: + use_json = isinstance(content, dict) temp_file = tempfile.NamedTemporaryFile( mode="w+t", delete=False, encoding="utf-8", + suffix=".json" if use_json else ".txt", ) temp_file.write( json.dumps(content) if isinstance(content, dict) else content, ) temp_file.flush() + temp_file.close() temp_files.append(temp_file) yield [file.name for file in temp_files] finally: for temp_file in temp_files: with suppress(Exception): - Path(temp_file.name).unlink() + temp_file.close() + try: + Path(temp_file.name).unlink(missing_ok=True) + except Exception as ex: + # Something went wrong and the file could not be deleted. Warn the user. + warnings.warn( + f"Failed to remove temporary file: '{temp_file.name}'. {ex}", + stacklevel=2, + ) diff --git a/airbyte/validate.py b/airbyte/validate.py index 4ecf2d1c..82da53a6 100644 --- a/airbyte/validate.py +++ b/airbyte/validate.py @@ -63,7 +63,7 @@ def full_tests(connector_name: str, sample_config: str) -> None: source = ab.get_source( # TODO: FIXME: noqa: SIM115, PTH123 connector_name, - config=json.load(open(sample_config, encoding="utf-8")), # noqa: SIM115, PTH123, + config=json.loads(Path(sample_config).read_text(encoding="utf-8")), # noqa: SIM115, PTH123, install_if_missing=False, ) @@ -118,8 +118,7 @@ def run() -> None: def validate(connector_dir: str, sample_config: str, *, validate_install_only: bool) -> None: # read metadata.yaml metadata_path = Path(connector_dir) / "metadata.yaml" - with Path(metadata_path).open(encoding="utf-8") as stream: - metadata = yaml.safe_load(stream)["data"] + metadata = yaml.safe_load(Path(metadata_path).read_text(encoding="utf-8"))["data"] # TODO: Use remoteRegistries.pypi.packageName once set for connectors connector_name = metadata["dockerRepository"].replace("airbyte/", "") @@ -147,15 +146,21 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b ], } - with tempfile.NamedTemporaryFile(mode="w+t", delete=True, encoding="utf-8") as temp_file: + with tempfile.NamedTemporaryFile( + mode="w+t", delete=True, encoding="utf-8", suffix="-catalog.json" + ) as temp_file: temp_file.write(json.dumps(registry)) temp_file.seek(0) os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name) - if validate_install_only: - install_only_test(connector_name) - else: - if not sample_config: - raise exc.PyAirbyteInputError( - input_value="--sample-config is required without --validate-install-only set" - ) - full_tests(connector_name, sample_config) + try: + if validate_install_only: + install_only_test(connector_name) + else: + if not sample_config: + raise exc.PyAirbyteInputError( + input_value="--sample-config is required without --validate-install-only set" + ) + full_tests(connector_name, sample_config) + finally: + # del os.environ["AIRBYTE_LOCAL_REGISTRY"] + temp_file.close() From c33e552d7bf7e357b418c944819e3b534a0c9fa4 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 12:12:05 -0700 Subject: [PATCH 08/30] flush file write instead of seeking --- airbyte/validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte/validate.py b/airbyte/validate.py index 82da53a6..48a13579 100644 --- a/airbyte/validate.py +++ b/airbyte/validate.py @@ -150,7 +150,7 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b mode="w+t", delete=True, encoding="utf-8", suffix="-catalog.json" ) as temp_file: temp_file.write(json.dumps(registry)) - temp_file.seek(0) + temp_file.flush() os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name) try: if validate_install_only: @@ -162,5 +162,5 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b ) full_tests(connector_name, sample_config) finally: - # del os.environ["AIRBYTE_LOCAL_REGISTRY"] + del os.environ["AIRBYTE_LOCAL_REGISTRY"] temp_file.close() From 06160aa0356f5a8487df6a2d7b4153bad5245216 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 12:14:21 -0700 Subject: [PATCH 09/30] fix lint issues --- airbyte/_processors/sql/duckdb.py | 1 - airbyte/_util/temp_files.py | 1 - airbyte/validate.py | 7 +++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte/_processors/sql/duckdb.py b/airbyte/_processors/sql/duckdb.py index fc6a8259..38b3a2e9 100644 --- a/airbyte/_processors/sql/duckdb.py +++ b/airbyte/_processors/sql/duckdb.py @@ -23,7 +23,6 @@ from sqlalchemy.engine import Engine - # @dataclass class DuckDBConfig(SqlConfig): """Configuration for DuckDB.""" diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index d36bcab2..da2273e7 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import os import tempfile import warnings from contextlib import contextmanager, suppress diff --git a/airbyte/validate.py b/airbyte/validate.py index 48a13579..31a87f5e 100644 --- a/airbyte/validate.py +++ b/airbyte/validate.py @@ -63,7 +63,7 @@ def full_tests(connector_name: str, sample_config: str) -> None: source = ab.get_source( # TODO: FIXME: noqa: SIM115, PTH123 connector_name, - config=json.loads(Path(sample_config).read_text(encoding="utf-8")), # noqa: SIM115, PTH123, + config=json.loads(Path(sample_config).read_text(encoding="utf-8")), # , install_if_missing=False, ) @@ -158,7 +158,10 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b else: if not sample_config: raise exc.PyAirbyteInputError( - input_value="--sample-config is required without --validate-install-only set" + input_value=( + "`--sample-config` is required when `--validate-install-only`" + "is not set." + ) ) full_tests(connector_name, sample_config) finally: From bd681c9da827e1cd581602392270be009ad59321 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 14:28:30 -0700 Subject: [PATCH 10/30] fix warning when streams checked from cloud --- airbyte/cloud/sync_results.py | 10 +++++++++- airbyte/datasets/_sql.py | 8 ++++++++ tests/integration_tests/cloud/test_cloud_sql_reads.py | 7 +++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/airbyte/cloud/sync_results.py b/airbyte/cloud/sync_results.py index 997566e7..89beab15 100644 --- a/airbyte/cloud/sync_results.py +++ b/airbyte/cloud/sync_results.py @@ -294,8 +294,16 @@ def get_dataset(self, stream_name: str) -> CachedDataset: """Retrieve an `airbyte.datasets.CachedDataset` object for a given stream name. This can be used to read and analyze the data in a SQL-based destination. + + TODO: In a future iteration, we can consider providing stream configuration information + (catalog information) to the `CachedDataset` object via the "Get stream properties" + API: https://reference.airbyte.com/reference/getstreamproperties """ - return CachedDataset(self.get_sql_cache(), stream_name=stream_name) + return CachedDataset( + self.get_sql_cache(), + stream_name=stream_name, + stream_configuration=False, # Don't look for stream configuration in cache. + ) def get_sql_database_name(self) -> str: """Return the SQL database name.""" diff --git a/airbyte/datasets/_sql.py b/airbyte/datasets/_sql.py index a5d36b29..b5e310d5 100644 --- a/airbyte/datasets/_sql.py +++ b/airbyte/datasets/_sql.py @@ -139,10 +139,17 @@ def __init__( self, cache: CacheBase, stream_name: str, + stream_configuration: ConfiguredAirbyteStream | None | Literal[False] = None, ) -> None: """We construct the query statement by selecting all columns from the table. This prevents the need to scan the table schema to construct the query statement. + + If stream_configuration is None, we attempt to retrieve the stream configuration from the + cache processor. This is useful when constructing a dataset from a CachedDataset object, + which already has the stream configuration. + + If stream_configuration is set to False, we skip the stream configuration retrieval. """ table_name = cache.processor.get_sql_table_name(stream_name) schema_name = cache.schema_name @@ -151,6 +158,7 @@ def __init__( cache=cache, stream_name=stream_name, query_statement=query, + stream_configuration=stream_configuration, ) @overrides diff --git a/tests/integration_tests/cloud/test_cloud_sql_reads.py b/tests/integration_tests/cloud/test_cloud_sql_reads.py index 6cb48afa..25e7b074 100644 --- a/tests/integration_tests/cloud/test_cloud_sql_reads.py +++ b/tests/integration_tests/cloud/test_cloud_sql_reads.py @@ -5,13 +5,12 @@ from contextlib import suppress +import airbyte as ab import pandas as pd import pytest -from sqlalchemy.engine.base import Engine - -import airbyte as ab from airbyte import cloud from airbyte.cloud.sync_results import SyncResult +from sqlalchemy.engine.base import Engine @pytest.fixture @@ -109,7 +108,7 @@ def test_read_from_deployed_connection( cache = sync_result.get_sql_cache() sqlalchemy_url = cache.get_sql_alchemy_url() engine: Engine = sync_result.get_sql_engine() - # assert sync_result.stream_names == ["users", "products", "purchases"] + assert "users" in sync_result.stream_names dataset: ab.CachedDataset = sync_result.get_dataset(stream_name="users") assert dataset.stream_name == "users" From 866f6e1187743314f6c348564ec0bce601df80b1 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 14:52:14 -0700 Subject: [PATCH 11/30] ignore warnings during docs generation --- tests/docs_tests/test_docs_checked_in.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/docs_tests/test_docs_checked_in.py b/tests/docs_tests/test_docs_checked_in.py index 868b488f..be2189a1 100644 --- a/tests/docs_tests/test_docs_checked_in.py +++ b/tests/docs_tests/test_docs_checked_in.py @@ -3,14 +3,17 @@ import os import docs.generate as generate +import pytest -def test_docs_checked_in(): +@pytest.mark.filterwarnings("ignore") +def test_docs_generation(): """ - Docs need to be generated via `poetry run generate-docs` and checked in to the repo. + Docs need to be able to be generated via `poetry run generate-docs`. - This test runs the docs generation and compares the output with the checked in docs. - It will fail if there are any differences. + This test runs the docs generation and ensures that it can complete successfully. + + Generation often produces warnings that are not relevant, so we suppress warnings in this test. """ generate.run() From 6116fb8261164caf92bfeb2c1fc621ba7b9feda8 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 14:52:33 -0700 Subject: [PATCH 12/30] ignore bigquery warning regarding JSON type (see issue #254) --- airbyte/_processors/sql/bigquery.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/airbyte/_processors/sql/bigquery.py b/airbyte/_processors/sql/bigquery.py index 3458c53a..182c72a8 100644 --- a/airbyte/_processors/sql/bigquery.py +++ b/airbyte/_processors/sql/bigquery.py @@ -3,6 +3,7 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import TYPE_CHECKING, Optional, final @@ -56,7 +57,17 @@ def dataset_name(self) -> str: @overrides def get_sql_alchemy_url(self) -> SecretString: - """Return the SQLAlchemy URL to use.""" + """Return the SQLAlchemy URL to use. + + We suppress warnings about unrecognized JSON type. More info on that here: + - https://github.com/airbytehq/PyAirbyte/issues/254 + """ + warnings.filterwarnings( + "ignore", + message="Did not recognize type 'JSON' of column", + category=sqlalchemy.exc.SAWarning, + ) + url: URL = make_url(f"bigquery://{self.project_name!s}") if self.credentials_path: url = url.update_query_dict({"credentials_path": self.credentials_path}) From 238e9b1d0984d46715694a386598f35002b9049b Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 15:12:31 -0700 Subject: [PATCH 13/30] ignore 'unraisable' warnings on windows (hard to repro) --- tests/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index ac14d7d4..fcfad50a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,9 @@ import shutil import socket import subprocess +import sys import time +import warnings from pathlib import Path import docker @@ -33,6 +35,14 @@ LOCAL_TEST_REGISTRY_URL = "./tests/integration_tests/fixtures/registry.json" +def pytest_configure(config): + if sys.platform.startswith("win"): + # Ignore specific warnings only on Windows + warnings.filterwarnings( + "ignore", category=pytest.PytestUnraisableExceptionWarning + ) + + def pytest_collection_modifyitems(items: list[Item]) -> None: """Override default pytest behavior, sorting our tests in a sensible execution order. From c5cd75bba11c991cb2f0788ea48d7dcabbc5257f Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 15:21:12 -0700 Subject: [PATCH 14/30] move check() out of fixtures --- tests/integration_tests/test_all_cache_types.py | 11 ++--------- tests/integration_tests/test_docker_executable.py | 14 ++------------ .../test_source_faker_integration.py | 14 ++------------ 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/tests/integration_tests/test_all_cache_types.py b/tests/integration_tests/test_all_cache_types.py index e7c4087f..f28f87cd 100644 --- a/tests/integration_tests/test_all_cache_types.py +++ b/tests/integration_tests/test_all_cache_types.py @@ -54,11 +54,8 @@ def source_faker_seed_a() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users"], ) - source.check() - source.select_streams([ - "users", - ]) return source @@ -74,11 +71,8 @@ def source_faker_seed_b() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users"], ) - source.check() - source.select_streams([ - "users", - ]) return source @@ -94,7 +88,6 @@ def source_pokeapi() -> ab.Source: install_if_missing=False, # Should already be on PATH streams="*", ) - source.check() return source diff --git a/tests/integration_tests/test_docker_executable.py b/tests/integration_tests/test_docker_executable.py index 4d5998fe..47f18a6e 100644 --- a/tests/integration_tests/test_docker_executable.py +++ b/tests/integration_tests/test_docker_executable.py @@ -42,13 +42,8 @@ def source_docker_faker_seed_a() -> ab.Source: "seed": SEED_A, "parallelism": 16, # Otherwise defaults to 4. }, + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "users", - "products", - "purchases", - ]) return source @@ -63,13 +58,8 @@ def source_docker_faker_seed_b() -> ab.Source: "seed": SEED_B, "parallelism": 16, # Otherwise defaults to 4. }, + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "products", - "purchases", - "users", - ]) return source diff --git a/tests/integration_tests/test_source_faker_integration.py b/tests/integration_tests/test_source_faker_integration.py index f31e65d2..6de903c2 100644 --- a/tests/integration_tests/test_source_faker_integration.py +++ b/tests/integration_tests/test_source_faker_integration.py @@ -67,13 +67,8 @@ def source_faker_seed_a() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "users", - "products", - "purchases", - ]) return source @@ -89,13 +84,8 @@ def source_faker_seed_b() -> ab.Source: "parallelism": 16, # Otherwise defaults to 4. }, install_if_missing=False, # Should already be on PATH + streams=["users", "products", "purchases"], ) - source.check() - source.select_streams([ - "products", - "purchases", - "users", - ]) return source From 6851765b4780a23793f1a0daf685c27740f6c645 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 15:32:35 -0700 Subject: [PATCH 15/30] revert (comment out) changes to temp file handling --- airbyte/_util/temp_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index da2273e7..89540bdd 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -21,18 +21,18 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, temp_files: list[Any] = [] try: for content in files_contents: - use_json = isinstance(content, dict) + # use_json = isinstance(content, dict) temp_file = tempfile.NamedTemporaryFile( mode="w+t", delete=False, encoding="utf-8", - suffix=".json" if use_json else ".txt", + # suffix=".json" if use_json else ".txt", ) temp_file.write( json.dumps(content) if isinstance(content, dict) else content, ) temp_file.flush() - temp_file.close() + # temp_file.close() temp_files.append(temp_file) yield [file.name for file in temp_files] finally: From 13026faefda8740c8974bd5996107e628235da69 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 15:49:58 -0700 Subject: [PATCH 16/30] add test of fixture setup logic --- tests/integration_tests/test_duckdb_cache.py | 24 +++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/integration_tests/test_duckdb_cache.py b/tests/integration_tests/test_duckdb_cache.py index 45ce7fe8..5022ccc1 100644 --- a/tests/integration_tests/test_duckdb_cache.py +++ b/tests/integration_tests/test_duckdb_cache.py @@ -8,15 +8,13 @@ from __future__ import annotations -from collections.abc import Generator import os -from pathlib import Path import sys - -import pytest - +from collections.abc import Generator +from pathlib import Path import airbyte as ab +import pytest from airbyte._executor import _get_bin_dir from airbyte.caches.duckdb import DuckDBCache from airbyte.caches.util import new_local_cache @@ -44,9 +42,8 @@ def add_venv_bin_to_path(monkeypatch): monkeypatch.setenv("PATH", new_path) -@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. -def source_faker() -> ab.Source: - """Fixture to return a source-faker connector instance.""" +def setup_source_faker() -> ab.Source: + """Test the source-faker setup.""" source = ab.get_source( "source-faker", local_executable="source-faker", @@ -66,6 +63,17 @@ def source_faker() -> ab.Source: return source +@pytest.fixture(scope="function") # Each test gets a fresh source-faker instance. +def source_faker() -> ab.Source: + """Fixture to return a source-faker connector instance.""" + return setup_source_faker() + + +def test_setup_source_faker() -> None: + """Test that fixture logic works as expected.""" + source = setup_source_faker() + + @pytest.fixture(scope="function") def duckdb_cache() -> Generator[DuckDBCache, None, None]: """Fixture to return a fresh cache.""" From 5e39f499ec6a2a4c62362f10f2b4d7c8dd125425 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 15:57:56 -0700 Subject: [PATCH 17/30] remove close() statement --- airbyte/_util/temp_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index 89540bdd..095000e9 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -37,8 +37,8 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, yield [file.name for file in temp_files] finally: for temp_file in temp_files: - with suppress(Exception): - temp_file.close() + # with suppress(Exception): + # temp_file.close() try: Path(temp_file.name).unlink(missing_ok=True) except Exception as ex: From d26ee8c80236fac143d53ee152a073fa19d5a5df Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 16:11:31 -0700 Subject: [PATCH 18/30] revert change to temp file handling --- airbyte/_util/temp_files.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index 095000e9..5e0ae892 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -37,13 +37,17 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, yield [file.name for file in temp_files] finally: for temp_file in temp_files: - # with suppress(Exception): - # temp_file.close() - try: - Path(temp_file.name).unlink(missing_ok=True) - except Exception as ex: - # Something went wrong and the file could not be deleted. Warn the user. - warnings.warn( - f"Failed to remove temporary file: '{temp_file.name}'. {ex}", - stacklevel=2, - ) + with suppress(Exception): + Path(temp_file.name).unlink() + + # TODO: Restore below after retrying the above reverted code + # # with suppress(Exception): + # # temp_file.close() + # try: + # Path(temp_file.name).unlink(missing_ok=True) + # except Exception as ex: + # # Something went wrong and the file could not be deleted. Warn the user. + # warnings.warn( + # f"Failed to remove temporary file: '{temp_file.name}'. {ex}", + # stacklevel=2, + # ) From 4bb6a3d2c7de4987dad82950f6a4ca87db6f45e8 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 16:53:41 -0700 Subject: [PATCH 19/30] ignore unclosed file issues (for now) --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 72d4111a..ded64ec1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,8 +86,11 @@ markers = [ # Treat python warnings as errors in pytest filterwarnings = [ "error", - "ignore::sqlalchemy.exc.MovedIn20Warning", "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", + # TODO: Remove these ignores once we are able to fix the warnings: + "ignore::sqlalchemy.exc.MovedIn20Warning", + "ignore::pytest.PytestUnraisableExceptionWarning", + "ignore:unclosed file:ResourceWarning", ] [tool.ruff] From 7ac225c82bfa26283c09cb46845698d33ee6811e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 18:36:25 -0700 Subject: [PATCH 20/30] remove colorama from explicit dependencies --- poetry.lock | 195 ++++++++++++++++++++++++------------------------- pyproject.toml | 1 - 2 files changed, 97 insertions(+), 99 deletions(-) diff --git a/poetry.lock b/poetry.lock index eda07897..cd0ed19c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -758,13 +758,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-cloud-bigquery" -version = "3.23.0" +version = "3.23.1" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.23.0.tar.gz", hash = "sha256:7ecdb207727d513b1bce1f213dbb926ed2e1d4f0122778de00f0e56d19d47a01"}, - {file = "google_cloud_bigquery-3.23.0-py2.py3-none-any.whl", hash = "sha256:dc0a4a47ab541a34aa1dc1f48539d88c091adc0637da7744d7fab6f3bc8886d5"}, + {file = "google-cloud-bigquery-3.23.1.tar.gz", hash = "sha256:4b4597f9291b42102c9667d3b4528f801d4c8f24ef2b12dd1ecb881273330955"}, + {file = "google_cloud_bigquery-3.23.1-py2.py3-none-any.whl", hash = "sha256:9fb72884fdbec9c4643cea6b7f21e1ecf3eb61d5305f87493d271dc801647a9e"}, ] [package.dependencies] @@ -1053,61 +1053,61 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "grpcio" -version = "1.63.0" +version = "1.64.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.63.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2e93aca840c29d4ab5db93f94ed0a0ca899e241f2e8aec6334ab3575dc46125c"}, - {file = "grpcio-1.63.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:91b73d3f1340fefa1e1716c8c1ec9930c676d6b10a3513ab6c26004cb02d8b3f"}, - {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:b3afbd9d6827fa6f475a4f91db55e441113f6d3eb9b7ebb8fb806e5bb6d6bd0d"}, - {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f3f6883ce54a7a5f47db43289a0a4c776487912de1a0e2cc83fdaec9685cc9f"}, - {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf8dae9cc0412cb86c8de5a8f3be395c5119a370f3ce2e69c8b7d46bb9872c8d"}, - {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:08e1559fd3b3b4468486b26b0af64a3904a8dbc78d8d936af9c1cf9636eb3e8b"}, - {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5c039ef01516039fa39da8a8a43a95b64e288f79f42a17e6c2904a02a319b357"}, - {file = "grpcio-1.63.0-cp310-cp310-win32.whl", hash = "sha256:ad2ac8903b2eae071055a927ef74121ed52d69468e91d9bcbd028bd0e554be6d"}, - {file = "grpcio-1.63.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2e44f59316716532a993ca2966636df6fbe7be4ab6f099de6815570ebe4383a"}, - {file = "grpcio-1.63.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:f28f8b2db7b86c77916829d64ab21ff49a9d8289ea1564a2b2a3a8ed9ffcccd3"}, - {file = "grpcio-1.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:65bf975639a1f93bee63ca60d2e4951f1b543f498d581869922910a476ead2f5"}, - {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b5194775fec7dc3dbd6a935102bb156cd2c35efe1685b0a46c67b927c74f0cfb"}, - {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4cbb2100ee46d024c45920d16e888ee5d3cf47c66e316210bc236d5bebc42b3"}, - {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff737cf29b5b801619f10e59b581869e32f400159e8b12d7a97e7e3bdeee6a2"}, - {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd1e68776262dd44dedd7381b1a0ad09d9930ffb405f737d64f505eb7f77d6c7"}, - {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:93f45f27f516548e23e4ec3fbab21b060416007dbe768a111fc4611464cc773f"}, - {file = "grpcio-1.63.0-cp311-cp311-win32.whl", hash = "sha256:878b1d88d0137df60e6b09b74cdb73db123f9579232c8456f53e9abc4f62eb3c"}, - {file = "grpcio-1.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:756fed02dacd24e8f488f295a913f250b56b98fb793f41d5b2de6c44fb762434"}, - {file = "grpcio-1.63.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:93a46794cc96c3a674cdfb59ef9ce84d46185fe9421baf2268ccb556f8f81f57"}, - {file = "grpcio-1.63.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a7b19dfc74d0be7032ca1eda0ed545e582ee46cd65c162f9e9fc6b26ef827dc6"}, - {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8064d986d3a64ba21e498b9a376cbc5d6ab2e8ab0e288d39f266f0fca169b90d"}, - {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:219bb1848cd2c90348c79ed0a6b0ea51866bc7e72fa6e205e459fedab5770172"}, - {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2d60cd1d58817bc5985fae6168d8b5655c4981d448d0f5b6194bbcc038090d2"}, - {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e350cb096e5c67832e9b6e018cf8a0d2a53b2a958f6251615173165269a91b0"}, - {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:56cdf96ff82e3cc90dbe8bac260352993f23e8e256e063c327b6cf9c88daf7a9"}, - {file = "grpcio-1.63.0-cp312-cp312-win32.whl", hash = "sha256:3a6d1f9ea965e750db7b4ee6f9fdef5fdf135abe8a249e75d84b0a3e0c668a1b"}, - {file = "grpcio-1.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:d2497769895bb03efe3187fb1888fc20e98a5f18b3d14b606167dacda5789434"}, - {file = "grpcio-1.63.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fdf348ae69c6ff484402cfdb14e18c1b0054ac2420079d575c53a60b9b2853ae"}, - {file = "grpcio-1.63.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a3abfe0b0f6798dedd2e9e92e881d9acd0fdb62ae27dcbbfa7654a57e24060c0"}, - {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6ef0ad92873672a2a3767cb827b64741c363ebaa27e7f21659e4e31f4d750280"}, - {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b416252ac5588d9dfb8a30a191451adbf534e9ce5f56bb02cd193f12d8845b7f"}, - {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3b77eaefc74d7eb861d3ffbdf91b50a1bb1639514ebe764c47773b833fa2d91"}, - {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b005292369d9c1f80bf70c1db1c17c6c342da7576f1c689e8eee4fb0c256af85"}, - {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdcda1156dcc41e042d1e899ba1f5c2e9f3cd7625b3d6ebfa619806a4c1aadda"}, - {file = "grpcio-1.63.0-cp38-cp38-win32.whl", hash = "sha256:01799e8649f9e94ba7db1aeb3452188048b0019dc37696b0f5ce212c87c560c3"}, - {file = "grpcio-1.63.0-cp38-cp38-win_amd64.whl", hash = "sha256:6a1a3642d76f887aa4009d92f71eb37809abceb3b7b5a1eec9c554a246f20e3a"}, - {file = "grpcio-1.63.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:75f701ff645858a2b16bc8c9fc68af215a8bb2d5a9b647448129de6e85d52bce"}, - {file = "grpcio-1.63.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cacdef0348a08e475a721967f48206a2254a1b26ee7637638d9e081761a5ba86"}, - {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:0697563d1d84d6985e40ec5ec596ff41b52abb3fd91ec240e8cb44a63b895094"}, - {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6426e1fb92d006e47476d42b8f240c1d916a6d4423c5258ccc5b105e43438f61"}, - {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e48cee31bc5f5a31fb2f3b573764bd563aaa5472342860edcc7039525b53e46a"}, - {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:50344663068041b34a992c19c600236e7abb42d6ec32567916b87b4c8b8833b3"}, - {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:259e11932230d70ef24a21b9fb5bb947eb4703f57865a404054400ee92f42f5d"}, - {file = "grpcio-1.63.0-cp39-cp39-win32.whl", hash = "sha256:a44624aad77bf8ca198c55af811fd28f2b3eaf0a50ec5b57b06c034416ef2d0a"}, - {file = "grpcio-1.63.0-cp39-cp39-win_amd64.whl", hash = "sha256:166e5c460e5d7d4656ff9e63b13e1f6029b122104c1633d5f37eaea348d7356d"}, - {file = "grpcio-1.63.0.tar.gz", hash = "sha256:f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1"}, + {file = "grpcio-1.64.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:3b09c3d9de95461214a11d82cc0e6a46a6f4e1f91834b50782f932895215e5db"}, + {file = "grpcio-1.64.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:7e013428ab472892830287dd082b7d129f4d8afef49227a28223a77337555eaa"}, + {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:02cc9cc3f816d30f7993d0d408043b4a7d6a02346d251694d8ab1f78cc723e7e"}, + {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f5de082d936e0208ce8db9095821361dfa97af8767a6607ae71425ac8ace15c"}, + {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b7bf346391dffa182fba42506adf3a84f4a718a05e445b37824136047686a1"}, + {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b2cbdfba18408389a1371f8c2af1659119e1831e5ed24c240cae9e27b4abc38d"}, + {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca4f15427d2df592e0c8f3d38847e25135e4092d7f70f02452c0e90d6a02d6d"}, + {file = "grpcio-1.64.0-cp310-cp310-win32.whl", hash = "sha256:7c1f5b2298244472bcda49b599be04579f26425af0fd80d3f2eb5fd8bc84d106"}, + {file = "grpcio-1.64.0-cp310-cp310-win_amd64.whl", hash = "sha256:73f84f9e5985a532e47880b3924867de16fa1aa513fff9b26106220c253c70c5"}, + {file = "grpcio-1.64.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2a18090371d138a57714ee9bffd6c9c9cb2e02ce42c681aac093ae1e7189ed21"}, + {file = "grpcio-1.64.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:59c68df3a934a586c3473d15956d23a618b8f05b5e7a3a904d40300e9c69cbf0"}, + {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b52e1ec7185512103dd47d41cf34ea78e7a7361ba460187ddd2416b480e0938c"}, + {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d598b5d5e2c9115d7fb7e2cb5508d14286af506a75950762aa1372d60e41851"}, + {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01615bbcae6875eee8091e6b9414072f4e4b00d8b7e141f89635bdae7cf784e5"}, + {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0b2dfe6dcace264807d9123d483d4c43274e3f8c39f90ff51de538245d7a4145"}, + {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7f17572dc9acd5e6dfd3014d10c0b533e9f79cd9517fc10b0225746f4c24b58e"}, + {file = "grpcio-1.64.0-cp311-cp311-win32.whl", hash = "sha256:6ec5ed15b4ffe56e2c6bc76af45e6b591c9be0224b3fb090adfb205c9012367d"}, + {file = "grpcio-1.64.0-cp311-cp311-win_amd64.whl", hash = "sha256:597191370951b477b7a1441e1aaa5cacebeb46a3b0bd240ec3bb2f28298c7553"}, + {file = "grpcio-1.64.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:1ce4cd5a61d4532651079e7aae0fedf9a80e613eed895d5b9743e66b52d15812"}, + {file = "grpcio-1.64.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:650a8150a9b288f40d5b7c1d5400cc11724eae50bd1f501a66e1ea949173649b"}, + {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8de0399b983f8676a7ccfdd45e5b2caec74a7e3cc576c6b1eecf3b3680deda5e"}, + {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46b8b43ba6a2a8f3103f103f97996cad507bcfd72359af6516363c48793d5a7b"}, + {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a54362f03d4dcfae63be455d0a7d4c1403673498b92c6bfe22157d935b57c7a9"}, + {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1f8ea18b928e539046bb5f9c124d717fbf00cc4b2d960ae0b8468562846f5aa1"}, + {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c56c91bd2923ddb6e7ed28ebb66d15633b03e0df22206f22dfcdde08047e0a48"}, + {file = "grpcio-1.64.0-cp312-cp312-win32.whl", hash = "sha256:874c741c8a66f0834f653a69e7e64b4e67fcd4a8d40296919b93bab2ccc780ba"}, + {file = "grpcio-1.64.0-cp312-cp312-win_amd64.whl", hash = "sha256:0da1d921f8e4bcee307aeef6c7095eb26e617c471f8cb1c454fd389c5c296d1e"}, + {file = "grpcio-1.64.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c46fb6bfca17bfc49f011eb53416e61472fa96caa0979b4329176bdd38cbbf2a"}, + {file = "grpcio-1.64.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3d2004e85cf5213995d09408501f82c8534700d2babeb81dfdba2a3bff0bb396"}, + {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6d5541eb460d73a07418524fb64dcfe0adfbcd32e2dac0f8f90ce5b9dd6c046c"}, + {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f279ad72dd7d64412e10f2443f9f34872a938c67387863c4cd2fb837f53e7d2"}, + {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85fda90b81da25993aa47fae66cae747b921f8f6777550895fb62375b776a231"}, + {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a053584079b793a54bece4a7d1d1b5c0645bdbee729215cd433703dc2532f72b"}, + {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:579dd9fb11bc73f0de061cab5f8b2def21480fd99eb3743ed041ad6a1913ee2f"}, + {file = "grpcio-1.64.0-cp38-cp38-win32.whl", hash = "sha256:23b6887bb21d77649d022fa1859e05853fdc2e60682fd86c3db652a555a282e0"}, + {file = "grpcio-1.64.0-cp38-cp38-win_amd64.whl", hash = "sha256:753cb58683ba0c545306f4e17dabf468d29cb6f6b11832e1e432160bb3f8403c"}, + {file = "grpcio-1.64.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:2186d76a7e383e1466e0ea2b0febc343ffeae13928c63c6ec6826533c2d69590"}, + {file = "grpcio-1.64.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0f30596cdcbed3c98024fb4f1d91745146385b3f9fd10c9f2270cbfe2ed7ed91"}, + {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:d9171f025a196f5bcfec7e8e7ffb7c3535f7d60aecd3503f9e250296c7cfc150"}, + {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf4c8daed18ae2be2f1fc7d613a76ee2a2e28fdf2412d5c128be23144d28283d"}, + {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3550493ac1d23198d46dc9c9b24b411cef613798dc31160c7138568ec26bc9b4"}, + {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3161a8f8bb38077a6470508c1a7301cd54301c53b8a34bb83e3c9764874ecabd"}, + {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8fabe2cc57a369638ab1ad8e6043721014fdf9a13baa7c0e35995d3a4a7618"}, + {file = "grpcio-1.64.0-cp39-cp39-win32.whl", hash = "sha256:31890b24d47b62cc27da49a462efe3d02f3c120edb0e6c46dcc0025506acf004"}, + {file = "grpcio-1.64.0-cp39-cp39-win_amd64.whl", hash = "sha256:5a56797dea8c02e7d3a85dfea879f286175cf4d14fbd9ab3ef2477277b927baa"}, + {file = "grpcio-1.64.0.tar.gz", hash = "sha256:257baf07f53a571c215eebe9679c3058a313fd1d1f7c4eede5a8660108c52d9c"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.63.0)"] +protobuf = ["grpcio-tools (>=1.64.0)"] [[package]] name = "grpcio-status" @@ -1270,13 +1270,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.59" +version = "0.1.60" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.59-py3-none-any.whl", hash = "sha256:445e3bc1d3baa1e5340cd979907a19483b9763a2ed37b863a01113d406f69345"}, - {file = "langsmith-0.1.59.tar.gz", hash = "sha256:e748a89f4dd6aa441349143e49e546c03b5dfb43376a25bfef6a5ca792fe1437"}, + {file = "langsmith-0.1.60-py3-none-any.whl", hash = "sha256:3c3520ea473de0a984237b3e9d638fdf23ef3acc5aec89a42e693225e72d6120"}, + {file = "langsmith-0.1.60.tar.gz", hash = "sha256:6a145b5454437f9e0f81525f23c4dcdbb8c07b1c91553b8f697456c418d6a599"}, ] [package.dependencies] @@ -1698,13 +1698,13 @@ files = [ [[package]] name = "pdoc" -version = "14.4.0" +version = "14.5.0" description = "API Documentation for Python Projects" optional = false python-versions = ">=3.8" files = [ - {file = "pdoc-14.4.0-py3-none-any.whl", hash = "sha256:6ea4fe07620b1f7601e2708a307a257636ec206e20b5611640b30f2e3cab47d6"}, - {file = "pdoc-14.4.0.tar.gz", hash = "sha256:c92edc425429ccbe287ace2a027953c24f13de53eab484c1a6d31ca72dd2fda9"}, + {file = "pdoc-14.5.0-py3-none-any.whl", hash = "sha256:9a8a84e19662610c0620fbe9f2e4174e3b090f8b601ed46348786ebb7517c508"}, + {file = "pdoc-14.5.0.tar.gz", hash = "sha256:79f534dc8a6494638dd6056b78e17a654df7ed34cc92646553ce3a7ba5a4fa4a"}, ] [package.dependencies] @@ -2111,13 +2111,13 @@ files = [ [[package]] name = "pytest" -version = "8.2.0" +version = "8.2.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"}, - {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"}, + {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, + {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, ] [package.dependencies] @@ -2357,13 +2357,13 @@ rpds-py = ">=0.7.0" [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -2593,19 +2593,18 @@ files = [ [[package]] name = "setuptools" -version = "69.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, - {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -2620,37 +2619,37 @@ files = [ [[package]] name = "snowflake-connector-python" -version = "3.10.0" +version = "3.10.1" description = "Snowflake Connector for Python" optional = false python-versions = ">=3.8" files = [ - {file = "snowflake_connector_python-3.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e2afca4bca70016519d1a7317c498f1d9c56140bf3e40ea40bddcc95fe827ca"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:d19bde29f89b226eb22af4c83134ecb5c229da1d5e960a01b8f495df78dcdc36"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bfe013ed97b4dd2e191fd6770a14030d29dd0108817d6ce76b9773250dd2d560"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0917c9f9382d830907e1a18ee1208537b203618700a9c671c2a20167b30f574"}, - {file = "snowflake_connector_python-3.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:7e828bc99240433e6552ac4cc4e37f223ae5c51c7880458ddb281668503c7491"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0d3d06d758455c50b998eabc1fd972a1f67faa5c85ef250fd5986f5a41aab0b"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:4602cb19b204bb03e03d65c6d5328467c9efc0fec53ca56768c3747c8dc8a70f"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb1a04b496bbd3e1e2e926df82b2369887b2eea958f535fb934c240bfbabf6c5"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c889f9f60f915d657e0a0ad2e6cc52cdcafd9bcbfa95a095aadfd8bcae62b819"}, - {file = "snowflake_connector_python-3.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e441484216ed416a6ed338133e23bd991ac4ba2e46531f4d330f61568c49314"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bb4aced19053c67513cecc92311fa9d3b507b2277698c8e987d404f6f3a49fb2"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:858315a2feff86213b079c6293ad8d850a778044c664686802ead8bb1337e1bc"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:adf16e1ca9f46d3bdf68e955ffa42075ebdb251e3b13b59003d04e4fea7d579a"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4c5c2a08b39086a5348502652ad4fdf24871d7ab30fd59f6b7b57249158468c"}, - {file = "snowflake_connector_python-3.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:05011286f42c52eb3e5a6db59ee3eaf79f3039f3a19d7ffac6f4ee143779c637"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:569301289ada5b0d72d0bd8432b7ca180220335faa6d9a0f7185f60891db6f2c"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:4e5641c70a12da9804b74f350b8cbbdffdc7aca5069b096755abd2a1fdcf5d1b"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ff767a1b8c48431549ac28884f8bd9647e63a23f470b05f6ab8d143c4b1475"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e52bbc1e2e7bda956525b4229d7f87579f8cabd7d5506b12aa754c4bcdc8c8d7"}, - {file = "snowflake_connector_python-3.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:280a8dcca0249e864419564e38764c08f8841900d9872fec2f2855fda494b29f"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:67bf570230b0cf818e6766c17245c7355a1f5ea27778e54ab8d09e5bb3536ad9"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:aa1e26f9c571d2c4206da5c978c1b345ffd798d3db1f9ae91985e6243c6bf94b"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73e9baa531d5156a03bfe5af462cf6193ec2a01cbb575edf7a2dd3b2a35254c7"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e03361c4749e4d65bf0d223fdea1c2d7a33af53b74e873929a6085d150aff17e"}, - {file = "snowflake_connector_python-3.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:e8cddd4357e70ab55d7aeeed144cbbeb1ff658b563d7d8d307afc06178a367ec"}, - {file = "snowflake_connector_python-3.10.0.tar.gz", hash = "sha256:7c7438e958753bd1174b73581d77c92b0b47a86c38d8ea0ba1ea23c442eb8e75"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69ccf052924995a77d6bdcf3de1b66ed90bf79ba150e02f614a5fc9a71812139"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:96bf0cc66e06ba9583cae412529ab2e0e38379a2f444c2978f25e7a033df5aad"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d011ee7f4d1461a90abe42d406e942c5089f34bcf151e81a89c5d14f195b474"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3c00a7109995df4379b3464e04ec39314ef82d2f64898323b0a5621f26af36c"}, + {file = "snowflake_connector_python-3.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:b22131059a6aabd3bcf6e3ed5eae52b8fcc5e4900a89c209c837107cb30b5ab2"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82f26632b2d0a9003a2c1ca1987294e73383a8a68bf260443706c506314a4b88"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:c1b76618f4607985f875dab264f3d281db2dcbf3db726e1a29b86aa22681dcf6"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d34fb7af5a14bd373928193b231647123b42df50f2f2069af0714c6bd1c99726"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424c7cca46daafec34a77a664f925b771fe38a31035373db61fc4134994a5501"}, + {file = "snowflake_connector_python-3.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:b1a8ae9df534891e2261cc8bac272bc8cc76b4e54b8b182e7d7a136cedb6e2e1"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cdd5bb449e0e6c493a6602ee2098ae71d77852acd39f28bf75ccf7ef4fcaca5"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:42e6d2a2327a8f25a6c68b4308a44d4fc442be0b3714805aadb0236fb17cc610"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86a676643953e0a460872dd52c67263a5ca29bea6e7020492b43ab5d99ebd48d"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a62150cb59b56b299e5715188bb244b233d0f16d56a2d26c35a7e04c3c40bbc"}, + {file = "snowflake_connector_python-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:8a4da738539a8c16baf7798e2f6382e3f29adf27988047de0445295b13eb9992"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5ee40801c4b1e3b8acd9c003e044d0275f7a49d56f96618fe164eb1245e6d9"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:2e59ca84acaa5309ced48ef6c8e41532f2c4665c084800737f406cbad2caee0d"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:886daab478fc704c1186cbed36f563cbe9623bcbe5dc9fe33e77cea17a40d9eb"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:480e9f586eaeec657ebe329ed7c6a5cde87f1331f5b2d0a672fb330cff6403ed"}, + {file = "snowflake_connector_python-3.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:7253a8f0e33d99f98f616b652a156a359f9c458b943639ae93e219718254ebaa"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bbe83707cf356285df2f3cdaf542d5416dc93d833c1c191ec896ba0c01f7c302"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:0edd33f090c40b6e60d524f4107b4c7a0353312766bf3a0a1c5a4165a87adf7f"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48bbc5c7b33a1e296732a97799448af4cf4b8a71bd5a1d0def667a418fe7f1e"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73a98c72a72119fd6ac3ac141ace55572727fd2ba4239e0abf82d4f7016630f1"}, + {file = "snowflake_connector_python-3.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:ea66d9a0a212452974f2a591a9ac0657e9982c0e9fa52e277372da4959d4ad21"}, + {file = "snowflake_connector_python-3.10.1.tar.gz", hash = "sha256:b9214da76ce72fff8eb60066fde6b9ba7f58a055d68ffe1e7a9b1034f57a34b4"}, ] [package.dependencies] @@ -3082,4 +3081,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "331322a76c543139350ddbb3dd716134527680ad10add1fe495bb6a596b84e6a" +content-hash = "93fe360cfb0bebbe09565548e650a03c3156a6c77e772160fdcb88cdf08475fd" diff --git a/pyproject.toml b/pyproject.toml index ded64ec1..0db19498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ ulid = "^1.1" # TODO: Remove this arbitrary python constraint once `sqlalchemy-bigquery` has done so. sqlalchemy-bigquery = { version = "1.9.0", python = "<3.13" } airbyte-api = "^0.49.2" -colorama = "^0.4.6" google-cloud-bigquery-storage = "^2.25.0" [tool.poetry.group.dev.dependencies] From ca8078f7ae05ee409e0aaa2958595566ec6d162d Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 18:39:00 -0700 Subject: [PATCH 21/30] remove comment --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0db19498..71ea169d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,6 @@ markers = [ filterwarnings = [ "error", "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", - # TODO: Remove these ignores once we are able to fix the warnings: "ignore::sqlalchemy.exc.MovedIn20Warning", "ignore::pytest.PytestUnraisableExceptionWarning", "ignore:unclosed file:ResourceWarning", From 77f48651e9b5fcd312b2f23d8b54bea56b8ba235 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 18:55:20 -0700 Subject: [PATCH 22/30] re-order ignores --- airbyte/_util/temp_files.py | 1 - pyproject.toml | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index 5e0ae892..0b7cb18b 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -5,7 +5,6 @@ import json import tempfile -import warnings from contextlib import contextmanager, suppress from pathlib import Path from typing import TYPE_CHECKING, Any diff --git a/pyproject.toml b/pyproject.toml index 71ea169d..dacd78c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,13 +82,15 @@ markers = [ "requires_creds: marks a test as requiring credentials (skip when secrets unavailable)", "linting: marks a test as a linting test", ] -# Treat python warnings as errors in pytest filterwarnings = [ + # Treat python warnings as errors in pytest "error", - "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", + # TODO: Remove these ignores once we are able to fix the warnings: "ignore::sqlalchemy.exc.MovedIn20Warning", - "ignore::pytest.PytestUnraisableExceptionWarning", "ignore:unclosed file:ResourceWarning", + "ignore::pytest.PytestUnraisableExceptionWarning", + # Order matters here. This must be last. + "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", ] [tool.ruff] From 3c3e731e15b703c00bca3d287741d3f6a49b7203 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 19:09:20 -0700 Subject: [PATCH 23/30] pin requests to avoid breaks --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index cd0ed19c..e39ea780 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2357,13 +2357,13 @@ rpds-py = ">=0.7.0" [[package]] name = "requests" -version = "2.32.2" +version = "2.31.0" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] [package.dependencies] @@ -3081,4 +3081,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "93fe360cfb0bebbe09565548e650a03c3156a6c77e772160fdcb88cdf08475fd" +content-hash = "6e3d7eeb9e986a14dd4c9936f6710e76973c2bd6e83e861ba7ab6209a4910a3b" diff --git a/pyproject.toml b/pyproject.toml index dacd78c9..4a0d9bd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ psycopg2-binary = "^2.9.9" pydantic = "<=2.0" python-dotenv = "^1.0.1" python-ulid = "^2.2.0" -requests = "^2.31.0" +requests = "<=2.31.0" # 3.32.0 breaks docker tests rich = "^13.7.0" snowflake-connector-python = "^3.10.0" snowflake-sqlalchemy = "^1.5.1" From d56b8939697de2ce66ceb9e00d160e2d4b344f14 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 19:27:42 -0700 Subject: [PATCH 24/30] improve comment --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4a0d9bd0..9cf86a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,8 @@ filterwarnings = [ "ignore::sqlalchemy.exc.MovedIn20Warning", "ignore:unclosed file:ResourceWarning", "ignore::pytest.PytestUnraisableExceptionWarning", - # Order matters here. This must be last. + # Order matters here. This must be last because it will fail if we + # hit another warning during the import. "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", ] From 554024441e27a00e68bce2b813e3e3ae68e0ca44 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 19:28:35 -0700 Subject: [PATCH 25/30] improve cleanup handling --- airbyte/_util/temp_files.py | 40 +++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index 0b7cb18b..bd7d2345 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -5,6 +5,8 @@ import json import tempfile +import time +import warnings from contextlib import contextmanager, suppress from pathlib import Path from typing import TYPE_CHECKING, Any @@ -20,33 +22,37 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, temp_files: list[Any] = [] try: for content in files_contents: - # use_json = isinstance(content, dict) + use_json = isinstance(content, dict) temp_file = tempfile.NamedTemporaryFile( mode="w+t", delete=False, encoding="utf-8", - # suffix=".json" if use_json else ".txt", + suffix=".json" if use_json else ".txt", ) temp_file.write( json.dumps(content) if isinstance(content, dict) else content, ) temp_file.flush() - # temp_file.close() + temp_file.close() temp_files.append(temp_file) yield [file.name for file in temp_files] finally: for temp_file in temp_files: - with suppress(Exception): - Path(temp_file.name).unlink() - - # TODO: Restore below after retrying the above reverted code - # # with suppress(Exception): - # # temp_file.close() - # try: - # Path(temp_file.name).unlink(missing_ok=True) - # except Exception as ex: - # # Something went wrong and the file could not be deleted. Warn the user. - # warnings.warn( - # f"Failed to remove temporary file: '{temp_file.name}'. {ex}", - # stacklevel=2, - # ) + max_attempts = 5 + for attempt in range(max_attempts): + try: + with suppress(Exception): + temp_file.close() + + Path(temp_file.name).unlink(missing_ok=True) + + break # File was deleted successfully. Move on. + except Exception as ex: + if attempt < max_attempts - 1: + time.sleep(1) # File might not be closed yet. Wait and try again. + else: + # Something went wrong and the file could not be deleted. Warn the user. + warnings.warn( + f"Failed to remove temporary file: '{temp_file.name}'. {ex}", + stacklevel=2, + ) From a9b108a7c198d1d838e43c56ab98cedb09256782 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 19:30:00 -0700 Subject: [PATCH 26/30] try without extra ignores --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9cf86a00..1ab6f4ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,8 +87,6 @@ filterwarnings = [ "error", # TODO: Remove these ignores once we are able to fix the warnings: "ignore::sqlalchemy.exc.MovedIn20Warning", - "ignore:unclosed file:ResourceWarning", - "ignore::pytest.PytestUnraisableExceptionWarning", # Order matters here. This must be last because it will fail if we # hit another warning during the import. "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning", From 189d7c7bce11020dd75263c5a86c6ef1cace8f1e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 19:43:53 -0700 Subject: [PATCH 27/30] don't close file --- airbyte/_util/temp_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte/_util/temp_files.py b/airbyte/_util/temp_files.py index bd7d2345..e1e1e91f 100644 --- a/airbyte/_util/temp_files.py +++ b/airbyte/_util/temp_files.py @@ -33,7 +33,8 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any, json.dumps(content) if isinstance(content, dict) else content, ) temp_file.flush() - temp_file.close() + # Don't close the file yet (breaks Windows) + # temp_file.close() temp_files.append(temp_file) yield [file.name for file in temp_files] finally: From 78f8c79594edfb34eb8ba444a6da0bf914efa366 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Wed, 22 May 2024 02:48:18 +0000 Subject: [PATCH 28/30] Auto-commit Updating dependencies Resolving dependencies... changes --- poetry.lock | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index e39ea780..042c2530 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-api" @@ -2303,6 +2303,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2310,8 +2311,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2328,6 +2337,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2335,6 +2345,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, From e8d9c7ceef6953d0109a04463a209fb33309bb05 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 20:07:13 -0700 Subject: [PATCH 29/30] windows-specific ignores --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index fcfad50a..0e176b24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,13 @@ def pytest_configure(config): if sys.platform.startswith("win"): # Ignore specific warnings only on Windows warnings.filterwarnings( - "ignore", category=pytest.PytestUnraisableExceptionWarning + "ignore", + category=pytest.PytestUnraisableExceptionWarning, + ) + warnings.filterwarnings( + "ignore", + category=ResourceWarning, + message="unclosed file", ) From 8ac9241bb1f3206c6e4d63cc95e81d644a763b34 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 21 May 2024 20:07:21 -0700 Subject: [PATCH 30/30] make ignores global --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1ab6f4ff..d78de9f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,9 @@ filterwarnings = [ "error", # TODO: Remove these ignores once we are able to fix the warnings: "ignore::sqlalchemy.exc.MovedIn20Warning", + # Ignore these, specifically on Windows because of file cleanup and locking issues: + "ignore:unclosed file:ResourceWarning", + "ignore::pytest.PytestUnraisableExceptionWarning", # Order matters here. This must be last because it will fail if we # hit another warning during the import. "ignore::airbyte.exceptions.AirbyteExperimentalFeatureWarning",