From 82e9ba053a47d70c594a6a516d7fe736111a362a Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:11:55 -0400 Subject: [PATCH 01/10] docs(bigquery): add update-adc flag to gcloud auth login (#10172) --- docs/backends/bigquery.qmd | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/backends/bigquery.qmd b/docs/backends/bigquery.qmd index 1522221c8b37..966c100a2e6b 100644 --- a/docs/backends/bigquery.qmd +++ b/docs/backends/bigquery.qmd @@ -120,7 +120,13 @@ The simplest way to authenticate with the BigQuery backend is to use [Google's ` Once you have `gcloud` installed, you can authenticate to BigQuery (and other Google Cloud services) by running ```sh -gcloud auth login +gcloud auth login --update-adc +``` + +You will also likely want to configure a default project: + +```sh +gcloud config set core/project ``` For any authentication problems, or information on other ways of authenticating, From 1667f432019030483003ad884217b5da3a2da509 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 19 Sep 2024 14:36:34 -0500 Subject: [PATCH 02/10] fix(mssql): ensure `ibis.random()` generates a new value per call (#10173) --- ibis/backends/sql/compilers/mssql.py | 6 ++++++ .../test_sql/test_rewrite_context/mssql/out.sql | 2 +- .../mssql-random/out.sql | 4 ++-- ibis/backends/tests/test_generic.py | 2 +- ibis/backends/tests/test_numeric.py | 12 ++++++++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ibis/backends/sql/compilers/mssql.py b/ibis/backends/sql/compilers/mssql.py index 70215fed821a..dbb0e3f9fe2c 100644 --- a/ibis/backends/sql/compilers/mssql.py +++ b/ibis/backends/sql/compilers/mssql.py @@ -171,6 +171,12 @@ def to_sqlglot( def visit_RandomUUID(self, op, **_): return self.f.newid() + def visit_RandomScalar(self, op, **_): + # By default RAND() will generate the same value for all calls within a + # query. The standard way to work around this is to pass in a unique + # value per call, which `CHECKSUM(NEWID())` provides. + return self.f.rand(self.f.checksum(self.f.newid())) + def visit_StringLength(self, op, *, arg): """The MSSQL LEN function doesn't count trailing spaces. diff --git a/ibis/backends/tests/snapshots/test_sql/test_rewrite_context/mssql/out.sql b/ibis/backends/tests/snapshots/test_sql/test_rewrite_context/mssql/out.sql index a724af27afcb..00bca1a41f25 100644 --- a/ibis/backends/tests/snapshots/test_sql/test_rewrite_context/mssql/out.sql +++ b/ibis/backends/tests/snapshots/test_sql/test_rewrite_context/mssql/out.sql @@ -1,4 +1,4 @@ SELECT TOP 10 - NTILE(2) OVER (ORDER BY RAND() ASC) - 1 AS [new_col] + NTILE(2) OVER (ORDER BY RAND(CHECKSUM(NEWID())) ASC) - 1 AS [new_col] FROM [test] AS [t0] \ No newline at end of file diff --git a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-random/out.sql b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-random/out.sql index 6944df66595d..b4da4f0b08c4 100644 --- a/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-random/out.sql +++ b/ibis/backends/tests/snapshots/test_sql/test_selects_with_impure_operations_not_merged/mssql-random/out.sql @@ -6,7 +6,7 @@ SELECT FROM ( SELECT [t0].[x], - RAND() AS [y], - RAND() AS [z] + RAND(CHECKSUM(NEWID())) AS [y], + RAND(CHECKSUM(NEWID())) AS [z] FROM [t] AS [t0] ) AS [t1] \ No newline at end of file diff --git a/ibis/backends/tests/test_generic.py b/ibis/backends/tests/test_generic.py index a42c84ba0d17..fb5c126fb967 100644 --- a/ibis/backends/tests/test_generic.py +++ b/ibis/backends/tests/test_generic.py @@ -569,7 +569,7 @@ def test_order_by(backend, alltypes, df, key, df_kwargs): backend.assert_frame_equal(result, expected) -@pytest.mark.notimpl(["polars", "mssql", "druid"]) +@pytest.mark.notimpl(["polars", "druid"]) @pytest.mark.notimpl( ["risingwave"], raises=PsycoPg2InternalError, diff --git a/ibis/backends/tests/test_numeric.py b/ibis/backends/tests/test_numeric.py index d5c16e9d08ac..4d23680c7cf4 100644 --- a/ibis/backends/tests/test_numeric.py +++ b/ibis/backends/tests/test_numeric.py @@ -1309,6 +1309,18 @@ def test_random(con): assert 0 <= result <= 1 +@pytest.mark.notimpl(["polars"], raises=com.OperationNotDefinedError) +@pytest.mark.notimpl(["druid"], raises=PyDruidProgrammingError) +@pytest.mark.notimpl( + ["risingwave"], + raises=PsycoPg2InternalError, + reason="function random() does not exist", +) +def test_random_different_per_row(alltypes): + result = alltypes.select("int_col", rand_col=ibis.random()).execute() + assert result.rand_col.nunique() > 1 + + @pytest.mark.parametrize( ("ibis_func", "pandas_func"), [ From 5a2e6e8314d256b66002f473af55a3d955be9c73 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:47:31 -0400 Subject: [PATCH 03/10] docs(clickhouse): entry into the accursed (#10174) Random number generation is cursed in ClickHouse. --- docs/reference/cursed_knowledge.qmd | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/reference/cursed_knowledge.qmd b/docs/reference/cursed_knowledge.qmd index 73e8db1d6e05..d0042243d706 100644 --- a/docs/reference/cursed_knowledge.qmd +++ b/docs/reference/cursed_knowledge.qmd @@ -14,3 +14,13 @@ execution engines. * Impala's `LTRIM` and `RTRIM` functions accept a _set_ of whitespace (or other) characters to remove from the left-, and right-hand-side sides of the input string, but the `TRIM` function only removes _spaces_. + +## ClickHouse + +* [ClickHouse's random number generating + functions](https://clickhouse.com/docs/en/sql-reference/functions/random-functions) + are considered in [common subexpression + elimination](https://en.wikipedia.org/wiki/Common_subexpression_elimination), + so to get two unique random numbers, users must defeat that optimization. + This is done by passing **any** argument to those functions. It's left as an + exercise for the reader to figure out how to generate two unique inputs. From c96e6c059a14b4e18e56e7304519163847a924db Mon Sep 17 00:00:00 2001 From: Naty Clementi Date: Fri, 20 Sep 2024 05:32:02 -0400 Subject: [PATCH 04/10] chore: add governance paragraph and link to governance doc to README (#10177) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fa536947a76d..56a6b6411cf4 100644 --- a/README.md +++ b/README.md @@ -218,3 +218,7 @@ Ibis is an open source project and welcomes contributions from anyone in the com Join our community by interacting on GitHub or chatting with us on [Zulip](https://ibis-project.zulipchat.com/). For more information visit https://ibis-project.org/. + +## Governance + +The Ibis project is an [independently governed](https://github.com/ibis-project/governance/blob/main/governance.md) open source community project to build and maintain the portable Python dataframe library. Ibis has contributors across a range of data companies and institutions. From 6042bf4d39bfee18e321715928db813702e7886c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 06:02:45 -0400 Subject: [PATCH 05/10] fix(deps): update dependency sqlglot to >=23.4,<25.23 (#10176) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- requirements-dev.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4a6eb4a87f13..e2b70943c21c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6890,13 +6890,13 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlglot" -version = "25.21.3" +version = "25.22.0" description = "An easily customizable SQL parser and transpiler" optional = false python-versions = ">=3.7" files = [ - {file = "sqlglot-25.21.3-py3-none-any.whl", hash = "sha256:dc63b429b80a69f2240ef892f776830883667fc9d978984ab98e7ce07edb7057"}, - {file = "sqlglot-25.21.3.tar.gz", hash = "sha256:273a447f71434ab2f9a36b81a6327706369735a0756a61cd576ac6896a5086a4"}, + {file = "sqlglot-25.22.0-py3-none-any.whl", hash = "sha256:3f03239456d6e19a922f2f59504e6363e8ed6ff8bbf6972c167ee5bf8d32a957"}, + {file = "sqlglot-25.22.0.tar.gz", hash = "sha256:f7b9291556ac73301c1a72dffe1802c0c2bf56c9d223382b3cc4cdfc2b9c26b8"}, ] [package.extras] @@ -7897,4 +7897,4 @@ visualization = ["graphviz"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "7acc0d4effb3c7f5476d41411f0a84982ab618fde1912ff6a32092b5c14787bf" +content-hash = "3a61ef75272dd8c592f94823bba3865b7ef5f53ec8bd3ba0263dff7ec9de07ee" diff --git a/pyproject.toml b/pyproject.toml index f47c0574b714..1f5b99d89d92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ atpublic = ">=2.3,<6" parsy = ">=2,<3" python-dateutil = ">=2.8.2,<3" pytz = ">=2022.7" -sqlglot = ">=23.4,<25.22" +sqlglot = ">=23.4,<25.23" toolz = ">=0.11,<1" typing-extensions = ">=4.3.0,<5" numpy = { version = ">=1.23.2,<3", optional = true } diff --git a/requirements-dev.txt b/requirements-dev.txt index 44fb62df156c..c2528c4bf870 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -254,7 +254,7 @@ sortedcontainers==2.4.0 ; python_version >= "3.10" and python_version < "4.0" soupsieve==2.6 ; python_version >= "3.10" and python_version < "3.13" sphobjinv==2.3.1.1 ; python_version >= "3.10" and python_version < "3.13" sqlalchemy==2.0.34 ; python_version >= "3.10" and python_version < "3.13" -sqlglot==25.21.3 ; python_version >= "3.10" and python_version < "4.0" +sqlglot==25.22.0 ; python_version >= "3.10" and python_version < "4.0" stack-data==0.6.3 ; python_version >= "3.10" and python_version < "4.0" statsmodels==0.14.2 ; python_version >= "3.10" and python_version < "3.13" tabulate==0.9.0 ; python_version >= "3.10" and python_version < "3.13" From 40500739a50a852605b56b829c1cbf276573988d Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 20 Sep 2024 06:43:33 -0400 Subject: [PATCH 06/10] test(bigquery): ensure that quoting test generates unique table name (#10182) --- ibis/backends/bigquery/tests/system/test_connect.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ibis/backends/bigquery/tests/system/test_connect.py b/ibis/backends/bigquery/tests/system/test_connect.py index cd3eebd2f743..151bbb55f8aa 100644 --- a/ibis/backends/bigquery/tests/system/test_connect.py +++ b/ibis/backends/bigquery/tests/system/test_connect.py @@ -12,6 +12,7 @@ import ibis import ibis.common.exceptions as exc +from ibis.util import gen_name def test_repeated_project_name(project_id, credentials): @@ -246,5 +247,12 @@ def test_create_table_from_memtable_needs_quotes(project_id, dataset_id, credent project_id=project_id, dataset_id=dataset_id, credentials=credentials ) - con.create_table("region-table", schema=dict(its_always="str", quoting="int")) - con.drop_table("region-table") + name = gen_name("region-table") + schema = dict(its_always="str", quoting="int") + + t = con.create_table(name, schema=schema) + + try: + assert t.schema() == ibis.schema(schema) + finally: + con.drop_table(name) From 57f20c819dfc1b065d087a4cb1caa2d89a177fdf Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:04:55 -0400 Subject: [PATCH 07/10] chore(docs): clean up current pyodide build (#10180) --- .github/workflows/docs-preview.yml | 2 +- .github/workflows/ibis-docs-main.yml | 6 +++--- .github/workflows/ibis-docs-pr.yml | 6 +++--- docs/tutorials/browser/repl.qmd | 9 ++++++--- justfile | 27 +++++++++------------------ 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index dd8972e897df..114e262637fe 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -77,7 +77,7 @@ jobs: path: docs/**/.jupyter_cache - name: build docs - run: nix develop --ignore-environment --keep HOME -c just docs-build-all + run: nix develop '.#ibis311' --ignore-environment --keep HOME -c just docs-build-all - name: install netlify cli run: npm install -g netlify-cli diff --git a/.github/workflows/ibis-docs-main.yml b/.github/workflows/ibis-docs-main.yml index 15157a28d509..47eb31ba1f7e 100644 --- a/.github/workflows/ibis-docs-main.yml +++ b/.github/workflows/ibis-docs-main.yml @@ -67,10 +67,10 @@ jobs: path: docs/**/.jupyter_cache - name: build api docs - run: nix develop --ignore-environment -c just docs-apigen --verbose + run: nix develop '.#ibis311' --ignore-environment -c just docs-apigen --verbose - name: build docs - run: nix develop --ignore-environment --keep HOME -c just docs-render + run: nix develop '.#ibis311' --ignore-environment --keep HOME -c just docs-render - name: cache rendered notebooks uses: actions/cache/save@v4 @@ -79,7 +79,7 @@ jobs: path: docs/**/.jupyter_cache - name: build jupyterlite - run: nix develop --ignore-environment --keep HOME -c just build-jupyterlite + run: nix develop '.#ibis311' --ignore-environment --keep HOME -c just build-jupyterlite - name: check that all frozen computations were done before push run: git diff --exit-code --stat diff --git a/.github/workflows/ibis-docs-pr.yml b/.github/workflows/ibis-docs-pr.yml index 0551f57b183f..9c62413f5fd5 100644 --- a/.github/workflows/ibis-docs-pr.yml +++ b/.github/workflows/ibis-docs-pr.yml @@ -67,13 +67,13 @@ jobs: path: docs/**/.jupyter_cache - name: generate api docs - run: nix develop --ignore-environment -c just docs-apigen --verbose + run: nix develop '.#ibis311' --ignore-environment -c just docs-apigen --verbose - name: build docs - run: nix develop --ignore-environment --keep HOME -c just docs-render + run: nix develop '.#ibis311' --ignore-environment --keep HOME -c just docs-render - name: build jupyterlite - run: nix develop --ignore-environment --keep HOME -c just build-jupyterlite + run: nix develop '.#ibis311' --ignore-environment --keep HOME -c just build-jupyterlite - name: check that all frozen computations were done before push run: git diff --exit-code --stat diff --git a/docs/tutorials/browser/repl.qmd b/docs/tutorials/browser/repl.qmd index a987ddfb7de3..d70c17d550bd 100644 --- a/docs/tutorials/browser/repl.qmd +++ b/docs/tutorials/browser/repl.qmd @@ -18,9 +18,12 @@ from urllib.parse import urlencode lines = """ %pip install numpy pandas tzdata import pyodide_js, pathlib, js -await pyodide_js.loadPackage("https://storage.googleapis.com/ibis-wasm-wheels/pyarrow-16.0.0.dev2661%2Bg9bddb87fd-cp311-cp311-emscripten_3_1_46_wasm32.whl") -pathlib.Path("penguins.csv").write_text(await (await js.fetch("https://storage.googleapis.com/ibis-tutorial-data/penguins.csv")).text()) -del pyodide_js, pathlib, js +wheel_url = "https://storage.googleapis.com/ibis-wasm-wheels/pyarrow-17.0.0-cp311-cp311-emscripten_3_1_46_wasm32.whl" +await pyodide_js.loadPackage(wheel_url) +penguins_csv_url = "https://storage.googleapis.com/ibis-tutorial-data/penguins.csv" +penguins_text = await (await js.fetch(penguins_csv_url)).text() +pathlib.Path("penguins.csv").write_text(penguins_text) +del pyodide_js, pathlib, js, wheel_url, penguins_csv_url, penguins_text %clear %pip install 'ibis-framework[duckdb]' from ibis.interactive import * diff --git a/justfile b/justfile index 777b45ca44c3..15e29c16898e 100644 --- a/justfile +++ b/justfile @@ -249,33 +249,24 @@ docs-api-preview: docs-deploy: quarto publish --no-prompt --no-browser --no-render netlify docs -# build an ibis_framework wheel that works with pyodide -build-ibis-for-pyodide: +# build jupyterlite repl +build-jupyterlite: #!/usr/bin/env bash set -euo pipefail - # TODO(cpcloud): remove when: - # 1. pyarrow release contains pyodide - # 2. ibis supports this version of pyarrow + mkdir -p docs/_output/jupyterlite + rm -rf dist/ - poetry add 'pyarrow>=10.0.1' --allow-prereleases + poetry-dynamic-versioning + ibis_dev_version="$(poetry version | cut -d ' ' -f2)" poetry build --format wheel - git checkout poetry.lock pyproject.toml - jq '{"PipliteAddon": {"piplite_urls": [$ibis, $duckdb]}}' -nM \ - --arg ibis dist/*.whl \ - --arg duckdb "https://duckdb.github.io/duckdb-pyodide/wheels/duckdb-1.0.0-cp311-cp311-emscripten_3_1_46_wasm32.whl" \ - > docs/jupyter_lite_config.json - -# build the jupyterlite deployment -build-jupyterlite: build-ibis-for-pyodide - #!/usr/bin/env bash - set -euo pipefail + git checkout pyproject.toml ibis/__init__.py - mkdir -p docs/_output/jupyterlite jupyter lite build \ --debug \ --no-libarchive \ - --config docs/jupyter_lite_config.json \ + --piplite-wheels "dist/ibis_framework-${ibis_dev_version}-py3-none-any.whl" \ + --piplite-wheels "https://duckdb.github.io/duckdb-pyodide/wheels/duckdb-1.1.0-cp311-cp311-emscripten_3_1_46_wasm32.whl" \ --apps repl \ --no-unused-shared-packages \ --output-dir docs/_output/jupyterlite From 28f7e0ba314e4a24d43ebf04c9bc5642ef4ba97d Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:29:44 -0400 Subject: [PATCH 08/10] ci: add automatic pull request labels (#10181) Adds some automatic labeling to incoming PRs based on files changed in the PR. --- .github/labeler.yml | 103 ++++++++++++++++++++++++++++++++++ .github/workflows/labeler.yml | 12 ++++ 2 files changed, 115 insertions(+) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000000..d49a58f89f5f --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,103 @@ +# backends +bigquery: + - changed-files: + - any-glob-to-any-file: "ibis/backends/bigquery/**" + +clickhouse: + - changed-files: + - any-glob-to-any-file: "ibis/backends/clickhouse/**" + +datafusion: + - changed-files: + - any-glob-to-any-file: "ibis/backends/datafusion/**" + +druid: + - changed-files: + - any-glob-to-any-file: "ibis/backends/druid/**" + +duckdb: + - changed-files: + - any-glob-to-any-file: "ibis/backends/duckdb/**" + +exasol: + - changed-files: + - any-glob-to-any-file: "ibis/backends/exasol/**" + +flink: + - changed-files: + - any-glob-to-any-file: "ibis/backends/flink/**" + +impala: + - changed-files: + - any-glob-to-any-file: "ibis/backends/impala/**" + +mssql: + - changed-files: + - any-glob-to-any-file: "ibis/backends/mssql/**" + +mysql: + - changed-files: + - any-glob-to-any-file: "ibis/backends/mysql/**" + +oracle: + - changed-files: + - any-glob-to-any-file: "ibis/backends/oracle/**" + +polars: + - changed-files: + - any-glob-to-any-file: "ibis/backends/polars/**" + +postgres: + - changed-files: + - any-glob-to-any-file: "ibis/backends/postgres/**" + +pyspark: + - changed-files: + - any-glob-to-any-file: "ibis/backends/pyspark/**" + +risingwave: + - changed-files: + - any-glob-to-any-file: "ibis/backends/risingwave/**" + +snowflake: + - changed-files: + - any-glob-to-any-file: "ibis/backends/snowflake/**" + +sqlite: + - changed-files: + - any-glob-to-any-file: "ibis/backends/sqlite/**" + +trino: + - changed-files: + - any-glob-to-any-file: "ibis/backends/trino/**" + +# miscellaneous labels +tests: + - changed-files: + - any-glob-to-any-file: "**/tests/**" + +nix: + - changed-files: + - any-glob-to-any-file: "**/*.nix" + - any-glob-to-any-file: "poetry.lock" + +datatypes: + - changed-files: + - any-glob-to-any-file: "ibis/expr/datatypes/**" + +ci: + - changed-files: + - any-glob-to-any-file: ".github/**" + +dependencies: + - changed-files: + - any-glob-to-any-file: "**/*.nix" + - any-glob-to-any-file: "poetry.lock" + - any-glob-to-any-file: "flake.lock" + - any-glob-to-any-file: "requirements-dev.txt" + - any-glob-to-any-file: "conda/*.yml" + +docs: + - changed-files: + - any-glob-to-any-file: "**/*.qmd" + - any-glob-to-any-file: "**/*.md" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 000000000000..596be479918b --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,12 @@ +name: PR Labeler +on: + - pull_request_target + +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v5 From 9642182d2d732deee31dbdc734668c94a049c7eb Mon Sep 17 00:00:00 2001 From: Naty Clementi Date: Fri, 20 Sep 2024 09:36:08 -0400 Subject: [PATCH 09/10] refactor(api): remove schema (#10149) BREAKING CHANGE: Removed hierarchical usage of schema. Ibis uses the following naming conventions: - schema: a mapping of column names to datatypes - database: a collection of tables - catalog: a collection of databases --------- Co-authored-by: Gil Forsyth Co-authored-by: Gil Forsyth --- docs/backends/_utils.py | 5 +- ibis/backends/__init__.py | 38 ---------- ibis/backends/bigquery/__init__.py | 64 +++++------------ .../bigquery/tests/system/test_client.py | 26 +++---- ibis/backends/conftest.py | 9 --- ibis/backends/datafusion/__init__.py | 14 ++-- ibis/backends/duckdb/__init__.py | 17 ++--- ibis/backends/duckdb/tests/test_client.py | 12 ++-- ibis/backends/exasol/__init__.py | 4 +- ibis/backends/mssql/__init__.py | 10 ++- ibis/backends/mssql/tests/test_client.py | 8 +-- ibis/backends/mysql/__init__.py | 19 +---- ibis/backends/mysql/tests/test_client.py | 5 +- ibis/backends/oracle/__init__.py | 23 ++----- ibis/backends/oracle/tests/test_client.py | 9 +-- ibis/backends/postgres/__init__.py | 24 ++----- ibis/backends/snowflake/__init__.py | 14 ++-- ibis/backends/snowflake/tests/test_client.py | 10 +-- ibis/backends/sql/__init__.py | 69 +++---------------- ibis/backends/sqlite/__init__.py | 5 -- ibis/backends/tests/test_client.py | 27 -------- ibis/backends/tests/test_signatures.py | 25 +------ ibis/backends/trino/__init__.py | 9 +-- ibis/backends/trino/tests/test_client.py | 10 +-- 24 files changed, 83 insertions(+), 373 deletions(-) diff --git a/docs/backends/_utils.py b/docs/backends/_utils.py index e54de2ce120e..53fad59bf176 100644 --- a/docs/backends/_utils.py +++ b/docs/backends/_utils.py @@ -40,10 +40,7 @@ def find_member_with_docstring(member): if base not in resolved_bases: resolved_bases.append(base) - # Remove `CanCreateSchema` and `CanListSchema` since they are deprecated - # and we don't want to document their existence. - filtered_bases = filter(lambda x: "schema" not in x.name.lower(), resolved_bases) - for base in filtered_bases: + for base in resolved_bases: try: parent_member = get_callable(base, member.name) except KeyError: diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index 8b73d812a34b..f1c06f1a4adb 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -741,44 +741,6 @@ def drop_database( """ -# TODO: remove this for 10.0 -class CanListSchema: - @util.deprecated( - instead="Use `list_databases` instead`", as_of="9.0", removed_in="10.0" - ) - def list_schemas( - self, like: str | None = None, database: str | None = None - ) -> list[str]: - return self.list_databases(like=like, catalog=database) - - @property - @util.deprecated( - instead="Use `Backend.current_database` instead.", - as_of="9.0", - removed_in="10.0", - ) - def current_schema(self) -> str: - return self.current_database - - -class CanCreateSchema(CanListSchema): - @util.deprecated( - instead="Use `create_database` instead", as_of="9.0", removed_in="10.0" - ) - def create_schema( - self, name: str, database: str | None = None, force: bool = False - ) -> None: - self.create_database(name=name, catalog=database, force=force) - - @util.deprecated( - instead="Use `drop_database` instead", as_of="9.0", removed_in="10.0" - ) - def drop_schema( - self, name: str, database: str | None = None, force: bool = False - ) -> None: - self.drop_database(name=name, catalog=database, force=force) - - class CacheEntry(NamedTuple): orig_op: ops.Relation cached_op_ref: weakref.ref[ops.Relation] diff --git a/ibis/backends/bigquery/__init__.py b/ibis/backends/bigquery/__init__.py index df1b2f3674d9..d87549394223 100644 --- a/ibis/backends/bigquery/__init__.py +++ b/ibis/backends/bigquery/__init__.py @@ -25,7 +25,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateDatabase, CanCreateSchema +from ibis.backends import CanCreateDatabase from ibis.backends.bigquery.client import ( bigquery_param, parse_project_and_dataset, @@ -155,7 +155,7 @@ def _postprocess_arrow( return table_or_batch.rename_columns(names) -class Backend(SQLBackend, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanCreateDatabase): name = "bigquery" compiler = sc.bigquery.compiler supports_python_udfs = False @@ -520,6 +520,10 @@ def disconnect(self) -> None: def _parse_project_and_dataset(self, dataset) -> tuple[str, str]: if isinstance(dataset, sge.Table): + if (sg_cat := dataset.args["catalog"]) is not None: + sg_cat.args["quoted"] = False + if (sg_db := dataset.args["db"]) is not None: + sg_db.args["quoted"] = False dataset = dataset.sql(self.dialect) if not dataset and not self.dataset: raise ValueError("Unable to determine BigQuery dataset.") @@ -582,9 +586,11 @@ def drop_database( self.raw_sql(stmt.sql(self.name)) def table( - self, name: str, database: str | None = None, schema: str | None = None + self, + name: str, + database: str | None = None, ) -> ir.Table: - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) table = sg.parse_one(f"`{name}`", into=sge.Table, read=self.name) # Bigquery, unlike other backends, had existing support for specifying @@ -612,10 +618,7 @@ def table( else: db = table.db - database = ( - sg.table(None, db=db, catalog=catalog, quoted=False).sql(dialect=self.name) - or None - ) + database = sg.table(None, db=db, catalog=catalog, quoted=False) or None project, dataset = self._parse_project_and_dataset(database) @@ -722,7 +725,6 @@ def insert( self, table_name: str, obj: pd.DataFrame | ir.Table | list | dict, - schema: str | None = None, database: str | None = None, overwrite: bool = False, ): @@ -734,15 +736,13 @@ def insert( The name of the table to which data needs will be inserted obj The source data or expression to insert - schema - The name of the schema that the table is located in database Name of the attached database that the table is located in. overwrite If `True` then replace existing contents of table """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) if catalog is None: catalog = self.current_catalog @@ -896,7 +896,6 @@ def list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, - schema: str | None = None, ) -> list[str]: """List the tables in the database. @@ -924,10 +923,8 @@ def list_tables( To specify a table in a separate BigQuery dataset, you can pass in the dataset and project as a string `"dataset.project"`, or as a tuple of strings `(dataset, project)`. - schema - [deprecated] The schema (dataset) inside `database` to perform the list against. """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) project, dataset = self._parse_project_and_dataset(table_loc) dataset_ref = bq.DatasetReference(project, dataset) @@ -1090,11 +1087,10 @@ def drop_table( self, name: str, *, - schema: str | None = None, database: tuple[str | str] | str | None = None, force: bool = False, ) -> None: - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) stmt = sge.Drop( kind="TABLE", @@ -1112,11 +1108,10 @@ def create_view( name: str, obj: ir.Table, *, - schema: str | None = None, database: str | None = None, overwrite: bool = False, ) -> ir.Table: - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) stmt = sge.Create( @@ -1137,11 +1132,10 @@ def drop_view( self, name: str, *, - schema: str | None = None, database: str | None = None, force: bool = False, ) -> None: - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) stmt = sge.Drop( @@ -1169,32 +1163,6 @@ def _register_udfs(self, expr: ir.Expr) -> None: def _safe_raw_sql(self, *args, **kwargs): yield self.raw_sql(*args, **kwargs) - # TODO: remove when the schema kwarg is removed - def _warn_and_create_table_loc(self, database=None, schema=None): - if schema is not None: - self._warn_schema() - if database is not None and schema is not None: - if isinstance(database, str): - table_loc = f"{database}.{schema}" - elif isinstance(database, tuple): - table_loc = database + schema - elif schema is not None: - table_loc = schema - elif database is not None: - table_loc = database - else: - table_loc = None - - table_loc = self._to_sqlglot_table(table_loc) - - if table_loc is not None: - if (sg_cat := table_loc.args["catalog"]) is not None: - sg_cat.args["quoted"] = False - if (sg_db := table_loc.args["db"]) is not None: - sg_db.args["quoted"] = False - - return table_loc - def compile(expr, params=None, **kwargs): """Compile an expression for BigQuery.""" diff --git a/ibis/backends/bigquery/tests/system/test_client.py b/ibis/backends/bigquery/tests/system/test_client.py index ad837065a949..df378c43e78d 100644 --- a/ibis/backends/bigquery/tests/system/test_client.py +++ b/ibis/backends/bigquery/tests/system/test_client.py @@ -34,6 +34,16 @@ def test_list_tables(con): tables = con.list_tables(like="functional_alltypes") assert set(tables) == {"functional_alltypes", "functional_alltypes_parted"} + pypi_tables = [ + "external", + "native", + ] + + assert con.list_tables() + + assert con.list_tables(database="ibis-gbq.pypi") == pypi_tables + assert con.list_tables(database=("ibis-gbq", "pypi")) == pypi_tables + def test_current_catalog(con): assert con.current_catalog == con.billing_project @@ -386,22 +396,6 @@ def test_create_table_with_options(con): con.drop_table(name) -def test_list_tables_schema_warning_refactor(con): - pypi_tables = [ - "external", - "native", - ] - - assert con.list_tables() - - # Warn but succeed for schema list - with pytest.raises(FutureWarning): - assert con.list_tables(schema="pypi") == pypi_tables - - assert con.list_tables(database="ibis-gbq.pypi") == pypi_tables - assert con.list_tables(database=("ibis-gbq", "pypi")) == pypi_tables - - def test_create_temp_table_from_scratch(project_id, dataset_id): con = ibis.bigquery.connect(project_id=project_id, dataset_id=dataset_id) name = gen_name("bigquery_temp_table") diff --git a/ibis/backends/conftest.py b/ibis/backends/conftest.py index 7b4ef991fee8..c1ca256bc638 100644 --- a/ibis/backends/conftest.py +++ b/ibis/backends/conftest.py @@ -17,7 +17,6 @@ from ibis.backends import ( CanCreateCatalog, CanCreateDatabase, - CanListSchema, _get_backend_names, ) from ibis.conftest import WINDOWS @@ -424,14 +423,6 @@ def con_no_data(backend_no_data): return backend_no_data.connection -@pytest.fixture(scope="session") -def con_list_schema(con): - if isinstance(con, CanListSchema): - return con - else: - pytest.skip(f"{con.name} backend cannot create schemas") - - @pytest.fixture(scope="session") def con_create_catalog(con): if isinstance(con, CanCreateCatalog): diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 0570f163e9f9..8ea440b3cdfa 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -22,7 +22,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateCatalog, CanCreateDatabase, CanCreateSchema, NoUrl +from ibis.backends import CanCreateCatalog, CanCreateDatabase, NoUrl from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C from ibis.common.dispatch import lazy_singledispatch @@ -69,7 +69,7 @@ def as_nullable(dtype: dt.DataType) -> dt.DataType: return dtype.copy(nullable=True) -class Backend(SQLBackend, CanCreateCatalog, CanCreateDatabase, CanCreateSchema, NoUrl): +class Backend(SQLBackend, CanCreateCatalog, CanCreateDatabase, NoUrl): name = "datafusion" supports_arrays = True compiler = sc.datafusion.compiler @@ -674,8 +674,10 @@ def create_table( return self.table(name, database=database) def truncate_table( - self, name: str, database: str | None = None, schema: str | None = None - ) -> None: + self, + name: str, + database: str | None = None, + ): """Delete all rows from a table. Parameters @@ -684,14 +686,12 @@ def truncate_table( Table name database Database name - schema - Schema name """ # datafusion doesn't support `TRUNCATE TABLE` so we use `DELETE FROM` # # however datafusion as of 34.0.0 doesn't implement DELETE DML yet - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) ident = sg.table(name, db=db, catalog=catalog).sql(self.dialect) diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index bb4d4f7aa9fd..0bdaa57a0f66 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -26,7 +26,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateDatabase, CanCreateSchema, UrlFromPath +from ibis.backends import CanCreateDatabase, UrlFromPath from ibis.backends.duckdb.converter import DuckDBPandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, C @@ -70,7 +70,7 @@ def __repr__(self): return repr(self.con.sql("from duckdb_settings()")) -class Backend(SQLBackend, CanCreateDatabase, CanCreateSchema, UrlFromPath): +class Backend(SQLBackend, CanCreateDatabase, UrlFromPath): name = "duckdb" compiler = sc.duckdb.compiler @@ -232,17 +232,13 @@ def create_table( return self.table(name, database=(catalog, database)) - def table( - self, name: str, schema: str | None = None, database: str | None = None - ) -> ir.Table: + def table(self, name: str, database: str | None = None) -> ir.Table: """Construct a table expression. Parameters ---------- name Table name - schema - [deprecated] Schema name database Database name @@ -252,7 +248,7 @@ def table( Table expression """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) # TODO: set these to better defaults catalog = table_loc.catalog or None @@ -987,7 +983,6 @@ def list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, - schema: str | None = None, ) -> list[str]: """List tables and views. @@ -1015,8 +1010,6 @@ def list_tables( To specify a table in a separate catalog, you can pass in the catalog and database as a string `"catalog.database"`, or as a tuple of strings `("catalog", "database")`. - schema - [deprecated] Schema name. If not passed, uses the current schema. Returns ------- @@ -1042,7 +1035,7 @@ def list_tables( ['baz'] """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog = table_loc.catalog or self.current_catalog database = table_loc.db or self.current_database diff --git a/ibis/backends/duckdb/tests/test_client.py b/ibis/backends/duckdb/tests/test_client.py index b4c357557f76..f02af4f040b6 100644 --- a/ibis/backends/duckdb/tests/test_client.py +++ b/ibis/backends/duckdb/tests/test_client.py @@ -78,13 +78,12 @@ def test_cross_db(tmpdir): con2.attach(path1, name="test1", read_only=True) - with pytest.warns(FutureWarning): - t1_from_con2 = con2.table("t1", schema="main", database="test1") + t1_from_con2 = con2.table("t1", database="test1.main") assert t1_from_con2.schema() == t2.schema() assert t1_from_con2.execute().equals(t2.execute()) - with pytest.warns(FutureWarning): - foo_t1_from_con2 = con2.table("t1", schema="foo", database="test1") + foo_t1_from_con2 = con2.table("t1", database="test1.foo") + assert foo_t1_from_con2.schema() == t2.schema() assert foo_t1_from_con2.execute().equals(t2.execute()) @@ -282,7 +281,7 @@ def test_invalid_connect(tmp_path): ibis.connect(url) -def test_list_tables_schema_warning_refactor(con): +def test_list_tables(con): assert { "astronauts", "awards_players", @@ -294,9 +293,6 @@ def test_list_tables_schema_warning_refactor(con): icecream_table = ["ice_cream"] - with pytest.warns(FutureWarning): - assert con.list_tables(schema="shops") == icecream_table - assert con.list_tables(database="shops") == icecream_table assert con.list_tables(database=("shops",)) == icecream_table diff --git a/ibis/backends/exasol/__init__.py b/ibis/backends/exasol/__init__.py index ee381b2b363c..b1bdb134bd67 100644 --- a/ibis/backends/exasol/__init__.py +++ b/ibis/backends/exasol/__init__.py @@ -18,7 +18,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateDatabase, CanCreateSchema +from ibis.backends import CanCreateDatabase from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, C @@ -36,7 +36,7 @@ _VARCHAR_REGEX = re.compile(r"^((VAR)?CHAR(?:\(\d+\)))?(?:\s+.+)?$") -class Backend(SQLBackend, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanCreateDatabase): name = "exasol" compiler = sc.exasol.compiler supports_temporary_tables = False diff --git a/ibis/backends/mssql/__init__.py b/ibis/backends/mssql/__init__.py index 3a215e1ece0b..d424d903179e 100644 --- a/ibis/backends/mssql/__init__.py +++ b/ibis/backends/mssql/__init__.py @@ -22,7 +22,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateCatalog, CanCreateDatabase, CanCreateSchema +from ibis.backends import CanCreateCatalog, CanCreateDatabase from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, C @@ -75,7 +75,7 @@ def datetimeoffset_to_datetime(value): # Databases: sys.schemas -class Backend(SQLBackend, CanCreateCatalog, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanCreateCatalog, CanCreateDatabase): name = "mssql" compiler = sc.mssql.compiler supports_create_or_replace = False @@ -527,7 +527,6 @@ def list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, - schema: str | None = None, ) -> list[str]: """List the tables in the database. @@ -552,10 +551,9 @@ def list_tables( To specify a table in a separate catalog, you can pass in the catalog and database as a string `"catalog.database"`, or as a tuple of strings `("catalog", "database")`. - schema - [deprecated] The schema inside `database` to perform the list against. + """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) sql = ( diff --git a/ibis/backends/mssql/tests/test_client.py b/ibis/backends/mssql/tests/test_client.py index 95b12c2972b7..3a72a75746c1 100644 --- a/ibis/backends/mssql/tests/test_client.py +++ b/ibis/backends/mssql/tests/test_client.py @@ -176,7 +176,7 @@ def test_glorious_length_function_hack(con, string): assert result == len(string) -def test_list_tables_schema_warning_refactor(con): +def test_list_tables(con): assert set(con.list_tables()) >= { "astronauts", "awards_players", @@ -188,12 +188,6 @@ def test_list_tables_schema_warning_refactor(con): restore_tables = ["restorefile", "restorefilegroup", "restorehistory"] - with pytest.warns(FutureWarning): - assert ( - con.list_tables(database="msdb", schema="dbo", like="restore") - == restore_tables - ) - assert con.list_tables(database="msdb.dbo", like="restore") == restore_tables assert con.list_tables(database=("msdb", "dbo"), like="restore") == restore_tables diff --git a/ibis/backends/mysql/__init__.py b/ibis/backends/mysql/__init__.py index c477c191ffb7..2490bc275ef3 100644 --- a/ibis/backends/mysql/__init__.py +++ b/ibis/backends/mysql/__init__.py @@ -313,7 +313,6 @@ def raw_sql(self, query: str | sg.Expression, **kwargs: Any) -> Any: def list_tables( self, like: str | None = None, - schema: str | None = None, database: tuple[str, str] | str | None = None, ) -> list[str]: """List the tables in the database. @@ -333,27 +332,11 @@ def list_tables( ---------- like A pattern to use for listing tables. - schema - [deprecated] The schema to perform the list against. database Database to list tables from. Default behavior is to show tables in the current database (`self.current_database`). """ - if schema is not None: - self._warn_schema() - - if schema is not None and database is not None: - raise ValueError( - "Using both the `schema` and `database` kwargs is not supported. " - "`schema` is deprecated and will be removed in Ibis 10.0" - "\nUse the `database` kwarg with one of the following patterns:" - '\ndatabase="database"' - '\ndatabase=("catalog", "database")' - '\ndatabase="catalog.database"', - ) - elif schema is not None: - table_loc = schema - elif database is not None: + if database is not None: table_loc = database else: table_loc = self.current_database diff --git a/ibis/backends/mysql/tests/test_client.py b/ibis/backends/mysql/tests/test_client.py index 95f4bd048c8e..831363d71362 100644 --- a/ibis/backends/mysql/tests/test_client.py +++ b/ibis/backends/mysql/tests/test_client.py @@ -226,7 +226,7 @@ def json_arrayagg(a) -> str: assert result == expected -def test_list_tables_schema_warning_refactor(con): +def test_list_tables(con): mysql_tables = { "column_stats", "columns_priv", @@ -236,9 +236,6 @@ def test_list_tables_schema_warning_refactor(con): } assert con.list_tables() - with pytest.warns(FutureWarning): - assert mysql_tables.issubset(con.list_tables(schema="mysql")) - assert mysql_tables.issubset(con.list_tables(database="mysql")) assert mysql_tables.issubset(con.list_tables(database=("mysql",))) diff --git a/ibis/backends/oracle/__init__.py b/ibis/backends/oracle/__init__.py index 6bcfe69b9591..b189ba3d3778 100644 --- a/ibis/backends/oracle/__init__.py +++ b/ibis/backends/oracle/__init__.py @@ -22,7 +22,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanListDatabase, CanListSchema +from ibis.backends import CanListDatabase from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import NULL, STAR, C @@ -75,7 +75,7 @@ def metadata_row_to_type( return typ -class Backend(SQLBackend, CanListDatabase, CanListSchema): +class Backend(SQLBackend, CanListDatabase): name = "oracle" compiler = sc.oracle.compiler @@ -270,7 +270,6 @@ def raw_sql(self, query: str | sg.Expression, **kwargs: Any) -> Any: def list_tables( self, like: str | None = None, - schema: str | None = None, database: tuple[str, str] | str | None = None, ) -> list[str]: """List the tables in the database. @@ -290,31 +289,17 @@ def list_tables( ---------- like A pattern to use for listing tables. - schema - [deprecated] The schema to perform the list against. database Database to list tables from. Default behavior is to show tables in the current database. """ - if schema is not None and database is not None: - raise exc.IbisInputError( - "Using both the `schema` and `database` kwargs is not supported. " - "`schema` is deprecated and will be removed in Ibis 10.0" - "\nUse the `database` kwarg with one of the following patterns:" - '\ndatabase="database"' - '\ndatabase=("catalog", "database")' - '\ndatabase="catalog.database"', - ) - if schema is not None: - # TODO: remove _warn_schema when the schema kwarg is removed - self._warn_schema() - table_loc = schema - elif database is not None: + if database is not None: table_loc = database else: table_loc = self.con.username.upper() + table_loc = self._to_sqlglot_table(table_loc) # Deeply frustrating here where if we call `convert` on `table_loc`, diff --git a/ibis/backends/oracle/tests/test_client.py b/ibis/backends/oracle/tests/test_client.py index 1806a2e0e1d4..97765b4d754f 100644 --- a/ibis/backends/oracle/tests/test_client.py +++ b/ibis/backends/oracle/tests/test_client.py @@ -8,7 +8,6 @@ import pytest import ibis -import ibis.common.exceptions as exc from ibis import udf from ibis.backends.oracle.tests.conftest import ( ORACLE_HOST, @@ -67,15 +66,9 @@ def stats_one_way_anova(x, y, value: str) -> int: tm.assert_frame_equal(result, expected, check_dtype=False) -def test_list_tables_schema_warning_refactor(con): +def test_list_tables(con): assert con.list_tables() - with pytest.raises(exc.IbisInputError): - con.list_tables(database="not none", schema="not none") - - with pytest.warns(FutureWarning): - assert con.list_tables(schema="SYS", like="EXU8OPT") == ["EXU8OPT"] - assert con.list_tables(database="SYS", like="EXU8OPT") == ["EXU8OPT"] diff --git a/ibis/backends/postgres/__init__.py b/ibis/backends/postgres/__init__.py index 68feacd6c02c..ddc3d4e169c4 100644 --- a/ibis/backends/postgres/__init__.py +++ b/ibis/backends/postgres/__init__.py @@ -21,7 +21,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateDatabase, CanCreateSchema, CanListCatalog +from ibis.backends import CanCreateDatabase, CanListCatalog from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import TRUE, C, ColGen, F @@ -35,7 +35,7 @@ import pyarrow as pa -class Backend(SQLBackend, CanListCatalog, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanListCatalog, CanCreateDatabase): name = "postgres" compiler = sc.postgres.compiler supports_python_udfs = True @@ -302,7 +302,6 @@ def _session_temp_db(self) -> str | None: def list_tables( self, like: str | None = None, - schema: str | None = None, database: tuple[str, str] | str | None = None, ) -> list[str]: """List the tables in the database. @@ -322,27 +321,12 @@ def list_tables( ---------- like A pattern to use for listing tables. - schema - [deprecated] The schema to perform the list against. database Database to list tables from. Default behavior is to show tables in the current database. """ - if schema is not None: - self._warn_schema() - - if schema is not None and database is not None: - raise ValueError( - "Using both the `schema` and `database` kwargs is not supported. " - "`schema` is deprecated and will be removed in Ibis 10.0" - "\nUse the `database` kwarg with one of the following patterns:" - '\ndatabase="database"' - '\ndatabase=("catalog", "database")' - '\ndatabase="catalog.database"', - ) - elif schema is not None: - table_loc = schema - elif database is not None: + + if database is not None: table_loc = database else: table_loc = (self.current_catalog, self.current_database) diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index 0d62bb76f438..5a53532f3e2f 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -27,7 +27,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateCatalog, CanCreateDatabase, CanCreateSchema +from ibis.backends import CanCreateCatalog, CanCreateDatabase from ibis.backends.snowflake.converter import SnowflakePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR @@ -137,7 +137,7 @@ } -class Backend(SQLBackend, CanCreateCatalog, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanCreateCatalog, CanCreateDatabase): name = "snowflake" compiler = sc.snowflake.compiler supports_python_udfs = True @@ -619,7 +619,6 @@ def list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, - schema: str | None = None, ) -> list[str]: """List the tables in the database. @@ -644,10 +643,8 @@ def list_tables( To specify a table in a separate Snowflake catalog, you can pass in the catalog and database as a string `"catalog.database"`, or as a tuple of strings `("catalog", "database")`. - schema - [deprecated] The schema inside `database` to perform the list against. """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) tables_query = "SHOW TABLES" views_query = "SHOW VIEWS" @@ -1181,7 +1178,6 @@ def insert( self, table_name: str, obj: pd.DataFrame | ir.Table | list | dict, - schema: str | None = None, database: str | None = None, overwrite: bool = False, ) -> None: @@ -1202,8 +1198,6 @@ def insert( The name of the table to which data needs will be inserted obj The source data or expression to insert - schema - [deprecated] The name of the schema that the table is located in database Name of the attached database that the table is located in. @@ -1214,7 +1208,7 @@ def insert( If `True` then replace existing contents of table """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) if not isinstance(obj, ir.Table): diff --git a/ibis/backends/snowflake/tests/test_client.py b/ibis/backends/snowflake/tests/test_client.py index 5e64c45a05bd..cd447104a25d 100644 --- a/ibis/backends/snowflake/tests/test_client.py +++ b/ibis/backends/snowflake/tests/test_client.py @@ -355,7 +355,7 @@ def test_struct_of_json(con): assert all(value == raw for value in result.to_pylist()) -def test_list_tables_schema_warning_refactor(con): +def test_list_tables(con): assert { "ASTRONAUTS", "AWARDS_PLAYERS", @@ -373,14 +373,6 @@ def test_list_tables_schema_warning_refactor(con): "TABLE_STORAGE_METRICS", ] - with pytest.warns(FutureWarning): - assert ( - con.list_tables( - database="IBIS_TESTING", schema="INFORMATION_SCHEMA", like="TABLE" - ) - == like_table - ) - assert ( con.list_tables(database="IBIS_TESTING.INFORMATION_SCHEMA", like="TABLE") == like_table diff --git a/ibis/backends/sql/__init__.py b/ibis/backends/sql/__init__.py index 2d64efe6076b..c2e4d47d5b76 100644 --- a/ibis/backends/sql/__init__.py +++ b/ibis/backends/sql/__init__.py @@ -25,47 +25,7 @@ from ibis.expr.schema import SchemaLike -class _DatabaseSchemaHandler: - """Temporary mixin collecting several helper functions and code snippets. - - Help to 'gracefully' deprecate the use of `schema` as a hierarchical term. - """ - - @staticmethod - def _warn_schema(): - util.warn_deprecated( - name="schema", - as_of="9.0", - removed_in="10.0", - instead="Use the `database` kwarg with one of the following patterns:" - '\ndatabase="database"' - '\ndatabase=("catalog", "database")' - '\ndatabase="catalog.database"', - # TODO: add option for namespace object - ) - - def _warn_and_create_table_loc(self, database=None, schema=None): - if schema is not None: - self._warn_schema() - - if database is not None and schema is not None: - if isinstance(database, str): - table_loc = f"{database}.{schema}" - elif isinstance(database, tuple): - table_loc = database + schema - elif schema is not None: - table_loc = schema - elif database is not None: - table_loc = database - else: - table_loc = None - - table_loc = self._to_sqlglot_table(table_loc) - - return table_loc - - -class SQLBackend(BaseBackend, _DatabaseSchemaHandler): +class SQLBackend(BaseBackend): compiler: ClassVar[SQLGlotCompiler] name: ClassVar[str] @@ -109,7 +69,6 @@ def _fetch_from_cursor(self, cursor, schema: sch.Schema) -> pd.DataFrame: def table( self, name: str, - schema: str | None = None, database: tuple[str, str] | str | None = None, ) -> ir.Table: """Construct a table expression. @@ -118,8 +77,6 @@ def table( ---------- name Table name - schema - [deprecated] Schema name database Database name @@ -129,7 +86,7 @@ def table( Table expression """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog = table_loc.catalog or None database = table_loc.db or None @@ -218,10 +175,9 @@ def create_view( obj: ir.Table, *, database: str | None = None, - schema: str | None = None, overwrite: bool = False, ) -> ir.Table: - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) src = sge.Create( @@ -240,10 +196,9 @@ def drop_view( name: str, *, database: str | None = None, - schema: str | None = None, force: bool = False, ) -> None: - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) src = sge.Drop( @@ -281,7 +236,7 @@ def drop_table( database: tuple[str, str] | str | None = None, force: bool = False, ) -> None: - table_loc = self._warn_and_create_table_loc(database, None) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) drop_stmt = sg.exp.Drop( @@ -358,7 +313,6 @@ def insert( self, table_name: str, obj: pd.DataFrame | ir.Table | list | dict, - schema: str | None = None, database: str | None = None, overwrite: bool = False, ) -> None: @@ -381,8 +335,6 @@ def insert( The name of the table to which data needs will be inserted obj The source data or expression to insert - schema - [deprecated] The name of the schema that the table is located in database Name of the attached database that the table is located in. @@ -393,7 +345,7 @@ def insert( If `True` then replace existing contents of table """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) if overwrite: @@ -485,7 +437,9 @@ def _build_insert_template( ).sql(self.dialect) def truncate_table( - self, name: str, database: str | None = None, schema: str | None = None + self, + name: str, + database: str | None = None, ) -> None: """Delete all rows from a table. @@ -509,11 +463,10 @@ def truncate_table( For backends that support multi-level table hierarchies, you can pass in a dotted string path like `"catalog.database"` or a tuple of strings like `("catalog", "database")`. - schema - [deprecated] Schema name + """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) catalog, db = self._to_catalog_db_tuple(table_loc) ident = sg.table(name, db=db, catalog=catalog, quoted=self.compiler.quoted).sql( diff --git a/ibis/backends/sqlite/__init__.py b/ibis/backends/sqlite/__init__.py index a1f4f078178d..a5c074aefbd8 100644 --- a/ibis/backends/sqlite/__init__.py +++ b/ibis/backends/sqlite/__init__.py @@ -560,13 +560,8 @@ def create_view( obj: ir.Table, *, database: str | None = None, - schema: str | None = None, overwrite: bool = False, ) -> ir.Table: - # schema was never used here, but warn for consistency - if schema is not None: - self._warn_schema() - view = sg.table(name, catalog=database, quoted=self.compiler.quoted) stmts = [] diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index f76474667684..069d6a525b2a 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -1375,33 +1375,6 @@ def test_create_database(con_create_database): assert database not in con_create_database.list_databases() -def test_list_schema_warns(con_list_schema): - with pytest.warns(FutureWarning): - con_list_schema.list_schemas() - - -@pytest.mark.never( - [ - "clickhouse", - "mysql", - "pyspark", - "flink", - ], - reason="No schema methods", -) -def test_create_schema(con_create_database): - schema = gen_name("test_create_schema") - with pytest.warns(FutureWarning): - con_create_database.create_schema(schema) - with pytest.warns(FutureWarning): - assert schema in con_create_database.list_schemas() - schema = schema.lower() - with pytest.warns(FutureWarning): - con_create_database.drop_schema(schema) - with pytest.warns(FutureWarning): - assert schema not in con_create_database.list_schemas() - - def test_list_databases(con_create_database): databases = con_create_database.list_databases() assert len(databases) == len(set(databases)) diff --git a/ibis/backends/tests/test_signatures.py b/ibis/backends/tests/test_signatures.py index 7d4095c44642..d1065867c0bf 100644 --- a/ibis/backends/tests/test_signatures.py +++ b/ibis/backends/tests/test_signatures.py @@ -82,7 +82,7 @@ def _scrape_methods(modules, params): "insert": pytest.param( SQLBackend, "insert", - marks=pytest.mark.notyet(["clickhouse", "flink", "impala", "sqlite"]), + marks=pytest.mark.notyet(["clickhouse", "flink", "impala"]), ), "list_databases": pytest.param( CanCreateDatabase, @@ -102,9 +102,7 @@ def _scrape_methods(modules, params): "list_tables": pytest.param( BaseBackend, "list_tables", - marks=pytest.mark.notyet( - ["flink", "mysql", "oracle", "postgres", "risingwave"] - ), + marks=pytest.mark.notyet(["flink"]), ), "read_csv": pytest.param( BaseBackend, @@ -131,21 +129,7 @@ def _scrape_methods(modules, params): "table", marks=pytest.mark.notyet( [ - "clickhouse", - "datafusion", - "druid", - "duckdb", - "exasol", - "mssql", - "mysql", - "oracle", "polars", - "postgres", - "risingwave", - "snowflake", - "sqlite", - "trino", - "pyspark", ] ), ), @@ -154,11 +138,6 @@ def _scrape_methods(modules, params): "to_parquet_dir", marks=pytest.mark.notyet(["pyspark"]), ), - "truncate_table": pytest.param( - SQLBackend, - "truncate_table", - marks=pytest.mark.notyet(["clickhouse", "impala"]), - ), } params = _scrape_methods( diff --git a/ibis/backends/trino/__init__.py b/ibis/backends/trino/__init__.py index 61c3c9a4b223..9ee1d628a007 100644 --- a/ibis/backends/trino/__init__.py +++ b/ibis/backends/trino/__init__.py @@ -20,7 +20,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends import CanCreateDatabase, CanCreateSchema, CanListCatalog +from ibis.backends import CanCreateDatabase, CanListCatalog from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import AlterTable, C @@ -35,7 +35,7 @@ import ibis.expr.operations as ops -class Backend(SQLBackend, CanListCatalog, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanListCatalog, CanCreateDatabase): name = "trino" compiler = sc.trino.compiler supports_create_or_replace = False @@ -213,7 +213,6 @@ def list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, - schema: str | None = None, ) -> list[str]: """List the tables in the database. @@ -230,10 +229,8 @@ def list_tables( To specify a table in a separate catalog, you can pass in the catalog and database as a string `"catalog.database"`, or as a tuple of strings `("catalog", "database")`. - schema - [deprecated] The schema inside `database` to perform the list against. """ - table_loc = self._warn_and_create_table_loc(database, schema) + table_loc = self._to_sqlglot_table(database) query = "SHOW TABLES" diff --git a/ibis/backends/trino/tests/test_client.py b/ibis/backends/trino/tests/test_client.py index 33a019c463e9..0efeb789b317 100644 --- a/ibis/backends/trino/tests/test_client.py +++ b/ibis/backends/trino/tests/test_client.py @@ -171,7 +171,7 @@ def test_table_access_database_schema(con): con.table("region", database="system.tpch.sf1") -def test_list_tables_schema_warning_refactor(con): +def test_list_tables(con): tpch_tables = [ "customer", "lineitem", @@ -185,14 +185,6 @@ def test_list_tables_schema_warning_refactor(con): assert con.list_tables() - # Error if user mixes tuple inputs and string inputs for database and schema - with pytest.raises(FutureWarning): - with pytest.raises(exc.IbisInputError): - con.list_tables(database=("tuple", "ohstuff"), schema="str") - - with pytest.warns(FutureWarning): - assert con.list_tables(database="tpch", schema="sf1") == tpch_tables - assert con.list_tables(database="tpch.sf1") == tpch_tables assert con.list_tables(database=("tpch", "sf1")) == tpch_tables From c62efce854c6e49f896bef565733744ee00ec156 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:13:04 -0400 Subject: [PATCH 10/10] docs(build): fetch all commits to enable proper dynamic versioning in the docs build (#10184) --- .github/workflows/ibis-docs-main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ibis-docs-main.yml b/.github/workflows/ibis-docs-main.yml index 47eb31ba1f7e..9e64a9624ca8 100644 --- a/.github/workflows/ibis-docs-main.yml +++ b/.github/workflows/ibis-docs-main.yml @@ -57,6 +57,8 @@ jobs: - name: checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: restore cache of the previously rendered notebooks uses: actions/cache/restore@v4