From 1d868791a23d9dcb1994bc6409a0ec86425f3f65 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Nov 2021 15:37:32 -0600 Subject: [PATCH 01/12] test: add session with prerelease dependencies --- noxfile.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ owlbot.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/noxfile.py b/noxfile.py index 2feeccdc..d831ca18 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import os import pathlib +import re import shutil import nox @@ -235,3 +236,86 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) diff --git a/owlbot.py b/owlbot.py index c69d54de..83e4ecd9 100644 --- a/owlbot.py +++ b/owlbot.py @@ -15,6 +15,7 @@ """This script is used to synthesize generated parts of this library.""" import pathlib +import re import synthtool as s from synthtool import gcp @@ -67,6 +68,99 @@ [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) +s.replaces( + ["noxfile.py"], + r'os.path.join\("docs", "_build", "html", ""\),\S+\)', + r""" + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) +""", + re.MULTILINE, +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From 2e6853100d8a35dc4d6a49bb3123d8888601259e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Nov 2021 16:29:54 -0600 Subject: [PATCH 02/12] fix owlbot --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index 83e4ecd9..4350ff3f 100644 --- a/owlbot.py +++ b/owlbot.py @@ -68,7 +68,7 @@ [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) -s.replaces( +s.replace( ["noxfile.py"], r'os.path.join\("docs", "_build", "html", ""\),\S+\)', r""" From 2d5e04b32db75d7caa21d6744f89389efe04ac3a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Nov 2021 16:34:28 -0600 Subject: [PATCH 03/12] add kokoro configs --- .kokoro/continuous/prerelease.cfg | 7 +++++++ .kokoro/presubmit/prerelease.cfg | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 .kokoro/continuous/prerelease.cfg create mode 100644 .kokoro/presubmit/prerelease.cfg diff --git a/.kokoro/continuous/prerelease.cfg b/.kokoro/continuous/prerelease.cfg new file mode 100644 index 00000000..00bc8678 --- /dev/null +++ b/.kokoro/continuous/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/.kokoro/presubmit/prerelease.cfg b/.kokoro/presubmit/prerelease.cfg new file mode 100644 index 00000000..00bc8678 --- /dev/null +++ b/.kokoro/presubmit/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} From c989c935cf55842077feee0c3349e60221aca65c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Nov 2021 16:46:18 -0600 Subject: [PATCH 04/12] match whitespace --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index 4350ff3f..2dd183ee 100644 --- a/owlbot.py +++ b/owlbot.py @@ -70,7 +70,7 @@ s.replace( ["noxfile.py"], - r'os.path.join\("docs", "_build", "html", ""\),\S+\)', + r'os.path.join\("docs", "_build", "html", ""\),\s+\)', r""" os.path.join("docs", "_build", "html", ""), ) From 187a4083b8a33a9b8ad58c32ffe216aebfe853f2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Nov 2021 16:47:32 -0600 Subject: [PATCH 05/12] remove extra whitespace --- owlbot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/owlbot.py b/owlbot.py index 2dd183ee..dd006390 100644 --- a/owlbot.py +++ b/owlbot.py @@ -71,8 +71,7 @@ s.replace( ["noxfile.py"], r'os.path.join\("docs", "_build", "html", ""\),\s+\)', - r""" - os.path.join("docs", "_build", "html", ""), + r"""os.path.join("docs", "_build", "html", ""), ) From fbb80f678a3bd339d2378516c7ce8a409004a73a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Nov 2021 16:59:20 -0600 Subject: [PATCH 06/12] escape in replacement --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index dd006390..fd5b9f86 100644 --- a/owlbot.py +++ b/owlbot.py @@ -131,7 +131,7 @@ def prerelease(session): deps = [ match.group(1) for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE ) ] From 15ec3430fc7933b976c03cb840fdc1cf870e2edd Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 29 Nov 2021 23:01:15 +0000 Subject: [PATCH 07/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- noxfile.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index d831ca18..cee6d88e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,6 @@ from __future__ import absolute_import import os import pathlib -import re import shutil import nox @@ -203,6 +202,89 @@ def docs(session): ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def docfx(session): """Build the docfx yaml files for this library.""" From 796baaf9264eba77e7ce74ec19573a1dd616bbcf Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Nov 2021 09:38:22 -0600 Subject: [PATCH 08/12] move nox session to avoid duplicates --- noxfile.py | 154 +++++++++++++---------------------------------------- owlbot.py | 25 +++++---- 2 files changed, 52 insertions(+), 127 deletions(-) diff --git a/noxfile.py b/noxfile.py index cee6d88e..df3378bf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import os import pathlib +import re import shutil import nox @@ -167,41 +168,6 @@ def system(session): ) -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session): - """Run the final coverage report. - - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=88") - - session.run("coverage", "erase") - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def docs(session): - """Build the docs for this library.""" - - session.install("-e", ".") - session.install("sphinx==4.0.1", "alabaster", "recommonmark") - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - @nox.session(python=DEFAULT_PYTHON_VERSION) def prerelease(session): session.install( @@ -285,6 +251,41 @@ def prerelease(session): ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=88") + + session.run("coverage", "erase") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx==4.0.1", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def docfx(session): """Build the docfx yaml files for this library.""" @@ -318,86 +319,3 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def prerelease(session): - session.install( - "--extra-index-url", - "https://pypi.fury.io/arrow-nightlies/", - "--prefer-binary", - "--pre", - "--upgrade", - "pyarrow", - ) - session.install( - "--extra-index-url", - "https://pypi.anaconda.org/scipy-wheels-nightly/simple", - "--prefer-binary", - "--pre", - "--upgrade", - "pandas", - ) - session.install( - "--prefer-binary", - "--pre", - "--upgrade", - "google-api-core", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", - "google-cloud-core", - "google-resumable-media", - "grpcio", - ) - session.install( - "freezegun", - "google-cloud-datacatalog", - "google-cloud-storage", - "google-cloud-testutils", - "IPython", - "mock", - "psutil", - "pytest", - "pytest-cov", - ) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") - - # Print out prerelease package versions. - session.run("python", "-m", "pip", "freeze") - - # Run all tests, except a few samples tests which require extra dependencies. - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", - os.path.join("tests", "unit"), - ) - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", - os.path.join("tests", "system"), - ) diff --git a/owlbot.py b/owlbot.py index fd5b9f86..ed0d1138 100644 --- a/owlbot.py +++ b/owlbot.py @@ -57,25 +57,24 @@ # ---------------------------------------------------------------------------- s.replace( - ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', + ["noxfile.py"], + r"import pathlib\s+import shutil", + "import pathlib\nimport re\nimport shutil", ) s.replace( - ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", + ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', ) + s.replace( - [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', + ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", ) s.replace( ["noxfile.py"], - r'os.path.join\("docs", "_build", "html", ""\),\s+\)', - r"""os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) + r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover(session):\s+" + r"""@nox.session(python=DEFAULT_PYTHON_VERSION) def prerelease(session): session.install( "--extra-index-url", @@ -156,10 +155,18 @@ def prerelease(session): f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", os.path.join("tests", "system"), ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): """, re.MULTILINE, ) +s.replace( + [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From aa4dba3621bfb76945c79a337ccb7fb6c9364be2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Nov 2021 09:43:32 -0600 Subject: [PATCH 09/12] escape more parens --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index ed0d1138..db4b3185 100644 --- a/owlbot.py +++ b/owlbot.py @@ -73,7 +73,7 @@ s.replace( ["noxfile.py"], - r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover(session):\s+" + r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):\s+" r"""@nox.session(python=DEFAULT_PYTHON_VERSION) def prerelease(session): session.install( From 226959ee90db9ee5be2b8d89c65fbd9d846f352d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Nov 2021 09:46:31 -0600 Subject: [PATCH 10/12] escape asterix --- owlbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index db4b3185..474c6a95 100644 --- a/owlbot.py +++ b/owlbot.py @@ -136,7 +136,7 @@ def prerelease(session): # We use --no-deps to ensure that pre-release versions aren't overwritten # by the version ranges in setup.py. - session.install(*deps) + session.install(\*deps) session.install("--no-deps", "-e", ".[all]") # Print out prerelease package versions. From b2eaafbee1238ddb43c277a1a8cf0009601a9f36 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Nov 2021 09:55:26 -0600 Subject: [PATCH 11/12] missing comma --- owlbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/owlbot.py b/owlbot.py index 474c6a95..e8a1b815 100644 --- a/owlbot.py +++ b/owlbot.py @@ -73,7 +73,7 @@ s.replace( ["noxfile.py"], - r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):\s+" + r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):\s+", r"""@nox.session(python=DEFAULT_PYTHON_VERSION) def prerelease(session): session.install( @@ -136,7 +136,7 @@ def prerelease(session): # We use --no-deps to ensure that pre-release versions aren't overwritten # by the version ranges in setup.py. - session.install(\*deps) + session.install(*deps) session.install("--no-deps", "-e", ".[all]") # Print out prerelease package versions. From 98800d6e9a65a7a7345a76a7253e307d4ff0fbfb Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Nov 2021 10:03:37 -0600 Subject: [PATCH 12/12] don't remove indent --- owlbot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/owlbot.py b/owlbot.py index e8a1b815..5ef93de7 100644 --- a/owlbot.py +++ b/owlbot.py @@ -73,7 +73,7 @@ s.replace( ["noxfile.py"], - r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):\s+", + r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", r"""@nox.session(python=DEFAULT_PYTHON_VERSION) def prerelease(session): session.install( @@ -158,8 +158,7 @@ def prerelease(session): @nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session): -""", +def cover(session):""", re.MULTILINE, )