diff --git a/.kokoro/continuous/prerelease.cfg b/.kokoro/continuous/prerelease.cfg new file mode 100644 index 00000000..00bc8678 --- /dev/null +++ b/.kokoro/continuous/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/.kokoro/presubmit/prerelease.cfg b/.kokoro/presubmit/prerelease.cfg new file mode 100644 index 00000000..00bc8678 --- /dev/null +++ b/.kokoro/presubmit/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/noxfile.py b/noxfile.py index 2feeccdc..df3378bf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import os import pathlib +import re import shutil import nox @@ -167,6 +168,89 @@ def system(session): ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. diff --git a/owlbot.py b/owlbot.py index c69d54de..5ef93de7 100644 --- a/owlbot.py +++ b/owlbot.py @@ -15,6 +15,7 @@ """This script is used to synthesize generated parts of this library.""" import pathlib +import re import synthtool as s from synthtool import gcp @@ -55,14 +56,112 @@ # Fixup files # ---------------------------------------------------------------------------- +s.replace( + ["noxfile.py"], + r"import pathlib\s+import shutil", + "import pathlib\nimport re\nimport shutil", +) + s.replace( ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', ) + s.replace( ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", ) +s.replace( + ["noxfile.py"], + r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", + r"""@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session):""", + re.MULTILINE, +) + s.replace( [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', )