diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index bc47660b..00000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,250 +0,0 @@ - -name: Build binary wheels and upload to PyPI - -on: - release: - types: [created] - schedule: - # Build nightly at 05:43 UTC - - cron: '43 5 * * *' - workflow_dispatch: # allow running manually from 'Actions' tab - -jobs: - build-source-distribution: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - - name: Build - run: python setup.py sdist - - uses: actions/upload-artifact@v3 - with: - name: python-package-distributions - path: dist/ - - build-wheels-linux: - name: Build wheels on ${{ matrix.os }} - ${{ matrix.arch }} - - strategy: - matrix: - os: - - ubuntu-20.04 - arch: - - x86_64 - - i686 - - aarch64 - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - - name: Set up QEMU for ARM64 cross compile - if: ${{ matrix.arch }} == 'aarch64' - id: qemu - uses: docker/setup-qemu-action@v3 - - - name: Build Wheels - uses: pypa/cibuildwheel@v2.16.2 - env: - CIBW_ARCHS_LINUX: ${{ matrix.arch }} - CIBW_ENVIRONMENT: > - CRC32C_PURE_PYTHON="0" - CRC32C_INSTALL_PREFIX="$(pwd)/usr" - CIBW_BUILD_VERBOSITY: 1 - # Build the C library inside CIBW so that the manylinux image is - # used to link the share library; otherwise, our extension wouldn't - # be able to link with it. - CIBW_BEFORE_BUILD: > - python -m pip install --require-hashes -r scripts/requirements.txt && - cmake -S google_crc32c -B build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCRC32C_BUILD_TESTS=no \ - -DCRC32C_BUILD_BENCHMARKS=no \ - -DBUILD_SHARED_LIBS=yes \ - -DCMAKE_INSTALL_PREFIX:PATH=$(pwd)/usr && - make -C build all install - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: py.test -v {project}/tests/ - - - uses: actions/upload-artifact@v3 - with: - name: python-package-distributions - path: wheelhouse/ - - build-wheels-macos: - name: Build wheels on ${{ matrix.os }} - ${{ matrix.arch }} - strategy: - matrix: - os: - - macos-10.15 - #- macos-11 - arch: - - x86_64 - - universal2 - runs-on: ${{ matrix.os }} - steps: - - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - - name: Get C library hash - id: get-c-lib-hash - run: - echo "::set-output name=hash::$(git -C google_crc32c log -n 1 --pretty=%H)" - - - id: load-cache - name: Load cached C library - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/usr - key: - libcrc32c-${{ matrix.os }}-${{ steps.get-c-lib-hash.outputs.hash }} - - - name: Build C Library - if: steps.load-cache.outputs.cache-hit != 'true' - run: > - python -m pip install --require-hashes -r scripts/requirements.txt && - cmake -S google_crc32c -B build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=10.9 \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ - -DCRC32C_BUILD_TESTS=no \ - -DCRC32C_BUILD_BENCHMARKS=no \ - -DBUILD_SHARED_LIBS=yes \ - -DCMAKE_INSTALL_PREFIX:PATH=${{ github.workspace }}/usr \ - -DCMAKE_INSTALL_NAME_DIR:PATH=${{ github.workspace }}/usr/lib && - make -C build all install - - - name: Build Wheels - uses: pypa/cibuildwheel@v2.16.2 - env: - CIBW_ARCHS_MACOS: ${{ matrix.arch }} - CIBW_ENVIRONMENT: > - CRC32C_PURE_PYTHON="0" - CRC32C_INSTALL_PREFIX="$(pwd)/usr" - CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: py.test -v {project}/tests/ - - - uses: actions/upload-artifact@v3 - with: - name: python-package-distributions - path: wheelhouse/ - - build-wheels-windows: - name: Build wheels on ${{ matrix.os }} ( ${{ matrix.platform }} ) - - strategy: - matrix: - os: - - windows-2019 - platform: - - x64 - - Win32 - - runs-on: ${{ matrix.os }} - - steps: - - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - - name: Get C library hash - id: get-c-lib-hash - run: - echo "::set-output name=hash::$(git -C google_crc32c log -n 1 --pretty=%H)" - - - id: load-cache - name: Load cached C library - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}\usr - key: - libcrc32c-${{ matrix.os }}-${{ matrix.platform }}-${{ steps.get-c-lib-hash.outputs.hash }} - - - name: Build C Library - if: steps.load-cache.outputs.cache-hit != 'true' - run: | - python -m pip install --require-hashes -r scripts/requirements.txt - cmake -S google_crc32c -B build -G "Visual Studio 16 2019" -A ${{ matrix.platform }} -DCRC32C_BUILD_BENCHMARKS=no -DCRC32C_BUILD_TESTS=no -DBUILD_SHARED_LIBS=no -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=yes -DCRC32C_USE_GLOG=0 - cmake --build build --verbose --config Release - cmake --install build --verbose --config Release --prefix=${{ github.workspace }}\usr\ - - # Passing through ${{ github.workspace }} to CIBW_ENVIRONMENT mangles - # backslashes: compute a variant which uses only forward-slashses. - - id: crc32c-install-prefix - name: Dead-reckon a CIBW-compatible install prefix - shell: bash - run: | - python -c "import os; workspace = '/'.join(os.getenv('GITHUB_WORKSPACE').split(os.sep)); pfx = f'{workspace}/usr'; print(f'::set-output name=prefix::{pfx}')" - - - id: platform-arch - name: Map platform -> wheel arch - shell: bash - run: | - if [[ "${{ matrix.platform }}" == "Win32" ]]; then - echo "::set-output name=arch::x86" - else - echo "::set-output name=arch::AMD64" - fi - - - name: Build Wheels - uses: pypa/cibuildwheel@v2.16.2 - env: - CIBW_ARCHS_WINDOWS: ${{ steps.platform-arch.outputs.arch }} - CIBW_ENVIRONMENT: > - CRC32C_INSTALL_PREFIX="${{ steps.crc32c-install-prefix.outputs.prefix }}" - CRC32C_PURE_PYTHON="0" - CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: py.test -v {project}/tests - - - uses: actions/upload-artifact@v3 - with: - name: python-package-distributions - path: wheelhouse/ - - - publish: - if: github.event_name == 'release' - needs: - - build-source-distribution - - build-wheels-linux - - build-wheels-macos - - build-wheels-windows - runs-on: ubuntu-latest - steps: - - name: Download all the dists - uses: actions/download-artifact@v3 - with: - name: python-package-distributions - path: dist/ - - name: Download all the wheels - uses: actions/download-artifact@v3 - with: - name: python-package-distributions - path: wheelhouse/ - - name: What will we publish? - run: ls wheelhouse/ - - name: Publish Source Distribution - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - packages_dir: dist/ - skip_existing: true - # repository_url: https://test.pypi.org/legacy/ - - name: Publish Wheels - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - packages_dir: wheelhouse/ - skip_existing: true - # repository_url: https://test.pypi.org/legacy/ diff --git a/.kokoro/build-manylinux.sh b/.kokoro/build-manylinux.sh new file mode 100755 index 00000000..057c5bc9 --- /dev/null +++ b/.kokoro/build-manylinux.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need enchant for spell check +sudo apt-get update +sudo apt-get -y install dictionaries-common aspell aspell-en \ + hunspell-en-us libenchant1c2a enchant + +set -eo pipefail + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +cd github/python-crc32c + +# Before running nox and such, build the extension. +./scripts/manylinux/build.sh +./scripts/manylinux/check.sh diff --git a/.kokoro/build-osx.sh b/.kokoro/build-osx.sh new file mode 100755 index 00000000..4e04cdb6 --- /dev/null +++ b/.kokoro/build-osx.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +cd github/python-crc32c + +# Before running nox and such, build the extension. +./scripts/osx/build.sh +./scripts/osx/check.sh diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 92ab3036..9b5bcc34 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -20,6 +20,14 @@ if [[ -z "${PROJECT_ROOT:-}" ]]; then fi cd "${PROJECT_ROOT}" +# Need enchant for spell check +sudo apt-get update +sudo apt-get -y install dictionaries-common aspell aspell-en \ + hunspell-en-us libenchant1c2a enchant + +set -eo pipefail + +cd github/python-crc32c # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 @@ -27,6 +35,9 @@ export PYTHONUNBUFFERED=1 # Debug: show build environment env | grep KOKORO +# Setup firestore account credentials +export FIRESTORE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/firebase-credentials.json + # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json diff --git a/.kokoro/continuous/crc32c.cfg b/.kokoro/continuous/crc32c.cfg new file mode 100644 index 00000000..495836ce --- /dev/null +++ b/.kokoro/continuous/crc32c.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} diff --git a/.kokoro/presubmit.bat b/.kokoro/presubmit.bat new file mode 100644 index 00000000..71c6511d --- /dev/null +++ b/.kokoro/presubmit.bat @@ -0,0 +1,32 @@ +@rem Copyright 2019 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo "Starting Windows build" + +cd /d %~dp0 +cd .. + +@rem as this package uses submodules make sure we have all content +call git config --global --add safe.directory C:/tmpfs/src/github/python-crc32c +call git submodule update --recursive || goto :error + +@echo "Build Wheels and Run Tests" +call scripts\windows\build.bat || goto :error + +for /r %%a in (*.whl) do xcopy "%%a" %KOKORO_ARTIFACTS_DIR% /i + +goto :EOF + +:error +exit /b 1 diff --git a/.kokoro/presubmit/linux/common.cfg b/.kokoro/presubmit/linux/common.cfg new file mode 100644 index 00000000..bf66b2f3 --- /dev/null +++ b/.kokoro/presubmit/linux/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/build.sh" +} diff --git a/.kokoro/presubmit/linux/presubmit-linux.cfg b/.kokoro/presubmit/linux/presubmit-linux.cfg new file mode 100644 index 00000000..c0f213b7 --- /dev/null +++ b/.kokoro/presubmit/linux/presubmit-linux.cfg @@ -0,0 +1,8 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} +build_file: "python-crc32c/.kokoro/build-manylinux.sh" diff --git a/.kokoro/presubmit/osx/common.cfg b/.kokoro/presubmit/osx/common.cfg new file mode 100644 index 00000000..bf66b2f3 --- /dev/null +++ b/.kokoro/presubmit/osx/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/build.sh" +} diff --git a/.kokoro/presubmit/osx/presubmit-osx-slcn.cfg b/.kokoro/presubmit/osx/presubmit-osx-slcn.cfg new file mode 100644 index 00000000..81c893d2 --- /dev/null +++ b/.kokoro/presubmit/osx/presubmit-osx-slcn.cfg @@ -0,0 +1,9 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/build-osx.sh" diff --git a/.kokoro/presubmit/osx/presubmit-osx.cfg b/.kokoro/presubmit/osx/presubmit-osx.cfg new file mode 100644 index 00000000..81c893d2 --- /dev/null +++ b/.kokoro/presubmit/osx/presubmit-osx.cfg @@ -0,0 +1,9 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/build-osx.sh" diff --git a/.kokoro/presubmit/windows/common.cfg b/.kokoro/presubmit/windows/common.cfg new file mode 100644 index 00000000..44a039a9 --- /dev/null +++ b/.kokoro/presubmit/windows/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/presubmit.bat" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/build.sh" +} diff --git a/.kokoro/presubmit/windows/presubmit-windows.cfg b/.kokoro/presubmit/windows/presubmit-windows.cfg new file mode 100644 index 00000000..495836ce --- /dev/null +++ b/.kokoro/presubmit/windows/presubmit-windows.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} diff --git a/.kokoro/release-manylinux-wheel.sh b/.kokoro/release-manylinux-wheel.sh new file mode 100755 index 00000000..f40b0a7f --- /dev/null +++ b/.kokoro/release-manylinux-wheel.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eo pipefail + +cd github/python-crc32c + +# Build for ManyLinux +./scripts/manylinux/build.sh + +REPO_ROOT=$(pwd) + +mkdir ${REPO_ROOT}/../pypi + +docker run \ + --rm \ + --interactive \ + --env REPO_ROOT=/var/code/python-crc32c/ \ + --volume ${REPO_ROOT}:/var/code/python-crc32c/ \ + --volume ${KOKORO_KEYSTORE_DIR}:/keys \ + quay.io/pypa/manylinux2014_x86_64 \ + /var/code/python-crc32c/.kokoro/release.sh diff --git a/.kokoro/release-osx-wheel.sh b/.kokoro/release-osx-wheel.sh new file mode 100755 index 00000000..2f376546 --- /dev/null +++ b/.kokoro/release-osx-wheel.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eo pipefail + +REPO_ROOT=$(pwd)/github/python-crc32c +cd github/python-crc32c + +# Build for OSX +./scripts/osx/build.sh + +# REPO_ROOT=$(pwd) +./.kokoro/release.sh diff --git a/.kokoro/release-windows-wheel.bat b/.kokoro/release-windows-wheel.bat new file mode 100755 index 00000000..d349e03f --- /dev/null +++ b/.kokoro/release-windows-wheel.bat @@ -0,0 +1,44 @@ +@rem Copyright 2019 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo "Starting Windows release" + +cd /d %~dp0 +cd .. + +@echo "Build Wheel" +call scripts\windows\build.bat || goto :error + +@echo "Start the releasetool reporter" +call py -3 -m pip install gcp-releasetool || goto :error +call py -3 -m releasetool publish-reporter-script > C:\temp\publisher-script || goto :error + +@echo "Ensure that we have the latest versions of Twine, Wheel, and Setuptools." +call py -3 -m pip install --upgrade twine wheel setuptools || goto :error + +@echo "Disable buffering, so that the logs stream through." +set PYTHONUNBUFFERED=1 + +@echo "## RELASE WORKFLOW SUCCESSFUL ##" +@echo "## Uploading Wheels ##" + +@echo "Move into the package, build the distribution and upload." +set /p TWINE_PASSWORD=<%KOKORO_KEYSTORE_DIR%/73713_google-cloud-pypi-token-keystore-1 +call py -3 setup.py sdist || goto :error +@REM call py -3 -m twine upload --skip-existing --username gcloudpypi --password "%TWINE_PASSWORD%" dist/* wheels/google_crc32c* || goto :error + +goto :EOF + +:error +exit /b 1 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 8e509855..6e7d411f 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -14,16 +14,59 @@ # limitations under the License. set -eo pipefail +echo "Release script started" + +if [ "$(uname)" == "Darwin" ]; then + # Mac OS + PYTHON=$(PYENV_VERSION=3.9 pyenv which python) + PYTHON_BIN=$(dirname ${PYTHON}) + + RELEASETOOL=${PYTHON_BIN}/releasetool + TWINE=${PYTHON_BIN}/twine + ${PYTHON} -m pip install gcp-releasetool twine --user + + echo "Change to code directory" + REPO_ROOT=$(pwd) + cd "${REPO_ROOT}" + ls + +else + # Kokoro Linux + mv /keys/73713_google-cloud-pypi-token-keystore-1 /73713_google-cloud-pypi-token-keystore-1 + + PATH=/opt/python/cp39-cp39/bin/:$PATH + PYTHON_BIN=/opt/python/cp39-cp39/bin/ + RELEASETOOL=${PYTHON_BIN}/releasetool + PYTHON=${PYTHON_BIN}/python + TWINE=${PYTHON_BIN}/twine + ${PYTHON} -m pip install gcp-releasetool twine + + echo "Change to code directory" + REPO_ROOT=/var/code/python-crc32c/ + cd "${REPO_ROOT}" + ls + +fi + +echo "Download dependencies for release script" # Start the releasetool reporter -python3 -m pip install --require-hashes -r github/python-crc32c/.kokoro/requirements.txt -python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script +${PYTHON} -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt +${RELEASETOOL} publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +# Ensure that we have the latest versions of Twine, Wheel, and Setuptools. +${PYTHON} -m pip install --upgrade twine wheel setuptools --user # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 - +echo "## RELASE WORKFLOW SUCCESSFUL ##" +echo "## Uploading Wheels ##" +# TODO: ONE OF THE BELOW WORKS # Move into the package, build the distribution and upload. TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-crc32c python3 setup.py sdist bdist_wheel -twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* +#twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* + +${PYTHON} setup.py sdist +# ${TWINE} upload --skip-existing --username gcloudpypi --password "${TWINE_PASSWORD}" dist/* wheels/* diff --git a/.kokoro/release/linux/common.cfg b/.kokoro/release/linux/common.cfg new file mode 100644 index 00000000..694320c7 --- /dev/null +++ b/.kokoro/release/linux/common.cfg @@ -0,0 +1,44 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} diff --git a/.kokoro/release/linux/linux_wheel.cfg b/.kokoro/release/linux/linux_wheel.cfg new file mode 100644 index 00000000..0c32ba5e --- /dev/null +++ b/.kokoro/release/linux/linux_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-manylinux-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-manylinux-wheel.sh" diff --git a/.kokoro/release/osx/common.cfg b/.kokoro/release/osx/common.cfg new file mode 100644 index 00000000..694320c7 --- /dev/null +++ b/.kokoro/release/osx/common.cfg @@ -0,0 +1,44 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} diff --git a/.kokoro/release/osx/osx_wheel.cfg b/.kokoro/release/osx/osx_wheel.cfg new file mode 100644 index 00000000..3a40d7a8 --- /dev/null +++ b/.kokoro/release/osx/osx_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-osx-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-osx-wheel.sh" diff --git a/.kokoro/release/windows/common.cfg b/.kokoro/release/windows/common.cfg new file mode 100644 index 00000000..694320c7 --- /dev/null +++ b/.kokoro/release/windows/common.cfg @@ -0,0 +1,44 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} diff --git a/.kokoro/release/windows/windows_wheel.cfg b/.kokoro/release/windows/windows_wheel.cfg new file mode 100644 index 00000000..1f640148 --- /dev/null +++ b/.kokoro/release/windows/windows_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-windows-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-windows-wheel.bat" diff --git a/CHANGELOG.md b/CHANGELOG.md index efb079d9..1e35adaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-crc32c/#history +## [1.6.0.dev1](https://github.com/googleapis/python-crc32c/compare/v1.5.0...v1.6.0.dev1) (2024-07-09) + + +### Features + +* Add support for python 3.12 ([#177](https://github.com/googleapis/python-crc32c/issues/177)) ([5ff1207](https://github.com/googleapis/python-crc32c/commit/5ff1207e7b60256e7a32932324ccb9ad4ec265d2)) + + +### Bug Fixes + +* Remove manylinux1 which is no longer supported by PyPA ([#186](https://github.com/googleapis/python-crc32c/issues/186)) ([79edb3f](https://github.com/googleapis/python-crc32c/commit/79edb3fd3cda0e4193a6fb6a8346058398df43de)) +* Update CI to use kokoro and build for OSX, Windows, Linux ([#214](https://github.com/googleapis/python-crc32c/pull/214)) + + ## [1.5.0](https://github.com/googleapis/python-crc32c/compare/v1.4.0...v1.5.0) (2022-08-31) diff --git a/README.md b/README.md index f59697f5..8b889fbf 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ C toolchain. # Currently Published Wheels -Wheels are currently published for Python 3.7, 3.8, 3.9, 3.10, 3.11 and -pypy-3.7 for multiple architectures. For information on building your own, -please view [BUILDING.md](BUILDING.md). +Wheels are currently published for CPython 3.8, 3.9, 3.10, 3.11 and 3.12 +for multiple architectures. PyPy 3.9 and 3.10 are also supported for Linux. +For information on building your own wheels please view [BUILDING.md](BUILDING.md). ## Linux @@ -28,9 +28,9 @@ See https://github.com/pypa/manylinux/issues/994. ## Mac OS -Wheels are published for `x86_64` and `universal2` architectures. +Wheels are published for `x86_64` and `arm64` architectures. ## Windows -Wheels are published for `win32` and `win_amd64` architectures. +Wheels are published for the `win_amd64` architecture. diff --git a/scripts/manylinux/build.sh b/scripts/manylinux/build.sh index 1b0e565f..c539bc77 100755 --- a/scripts/manylinux/build.sh +++ b/scripts/manylinux/build.sh @@ -23,9 +23,12 @@ REPO_ROOT=$(dirname ${SCRIPTS_DIR}) cd $REPO_ROOT -git submodule update --init --recursive +# Add directory as safe to avoid "detected dubious ownership" fatal issue1 +git config --global --add safe.directory $REPO_ROOT +git config --global --add safe.directory $REPO_ROOT/google_crc32c +git submodule update --init --recursive -docker pull quay.io/pypa/manylinux2010_x86_64 +docker pull quay.io/pypa/manylinux2010_x86_64 docker run \ --rm \ --interactive \ @@ -52,3 +55,13 @@ docker run \ --env BUILD_PYTHON=${BUILD_PYTHON} \ quay.io/pypa/manylinux2014_aarch64 \ /var/code/python-crc32c/scripts/manylinux/build_on_centos.sh + +echo "Build completed" + +# Upload wheels to GCS for debugging. Uncomment only when needed. + +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} +ls ${REPO_ROOT}/wheels/ +gsutil cp ${REPO_ROOT}/wheels/* gs://python_crc32c/ +echo "Linux wheels uploaded successfully" diff --git a/scripts/manylinux/build_on_centos.sh b/scripts/manylinux/build_on_centos.sh index e6d2871d..300b9f28 100755 --- a/scripts/manylinux/build_on_centos.sh +++ b/scripts/manylinux/build_on_centos.sh @@ -24,6 +24,8 @@ ${MAIN_PYTHON_BIN}/python -m pip install --upgrade pip # Install `cmake` (i.e. non-Python build dependency). ${MAIN_PYTHON_BIN}/python -m pip install "cmake >= 3.12.0" # Install Python build dependencies. +# cd ${REPO_ROOT} +# ${MAIN_PYTHON_BIN}/pip install . ${MAIN_PYTHON_BIN}/python -m pip install \ --requirement ${REPO_ROOT}/scripts/dev-requirements.txt @@ -45,10 +47,7 @@ if [[ -z ${BUILD_PYTHON} ]]; then # Collect all target Python versions. for PYTHON_BIN in /opt/python/*/bin; do # H/T: https://stackoverflow.com/a/229606/1068170 - if [[ "${PYTHON_BIN}" == *"37"* ]]; then - PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" - continue - elif [[ "${PYTHON_BIN}" == *"38"* ]]; then + if [[ "${PYTHON_BIN}" == *"38"* ]]; then PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" continue elif [[ "${PYTHON_BIN}" == *"39"* ]]; then @@ -57,6 +56,12 @@ if [[ -z ${BUILD_PYTHON} ]]; then elif [[ "${PYTHON_BIN}" == *"310"* ]]; then PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" continue + elif [[ "${PYTHON_BIN}" == *"311"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue + elif [[ "${PYTHON_BIN}" == *"312"* ]]; then + PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" + continue else echo "Ignoring unsupported version: ${PYTHON_BIN}" echo "=====================================" @@ -88,3 +93,5 @@ done # Clean up. rm -fr ${REPO_ROOT}/google_crc32c/build/ rm -fr ${REPO_ROOT}/dist_wheels/ + +echo "Centos Build completed" diff --git a/scripts/manylinux/check.sh b/scripts/manylinux/check.sh index 25db1687..73a3e139 100755 --- a/scripts/manylinux/check.sh +++ b/scripts/manylinux/check.sh @@ -20,25 +20,47 @@ echo "CHECKING ON LINUX" VERSION=$(awk "/version \= ([0-9.]+)/" setup.cfg) PACKAGE_VERSION=${VERSION:10} -WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" -PYTHON=python3.7 -# Using pyenv, set 3.7.13 as a local python version. -# pyenv versions -pyenv local 3.7.13 +# set up pyenv & shell environment for switching across python versions +eval "$(pyenv init -)" +eval "$(pyenv init --path)" +install_python_pyenv() { + version=$1 -# Make sure we can create a virtual environment. -${PYTHON} -m pip install --upgrade setuptools pip wheel + if [ -z "$(pyenv versions --bare | grep $version)" ]; then + echo "Python $version is not installed. Installing..." + pyenv install $version + echo "Python $version installed." + else + echo "Python $version is already installed." + fi + pyenv shell $version +} -# Create a virtual environment. -${PYTHON} -m venv venv +SUPPORTED_PYTHON_VERSIONS=("3.8" "3.9" "3.10" "3.11" "3.12") -# Install the wheel. -venv/bin/pip install ${WHEEL_FILE} +for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + PYTHON=python${PYTHON_VERSION} + install_python_pyenv ${PYTHON_VERSION} + ${PYTHON} -m pip install --upgrade setuptools pip wheel -# Verify that the module is installed and peek at contents. -venv/bin/python scripts/check_crc32c_extension.py -unzip -l ${WHEEL_FILE} + # Create a virtual environment. + ${PYTHON} -m venv venv -# Clean up. -rm -fr venv/ + # Note that the 'm' SOABI flag is no longer supported for Python >= 3.8 + SOABI_FLAG="m" + if [ "${PYTHON_VERSION}" != "3.7" ]; then + SOABI_FLAG="" + fi + + # Install the wheel. + WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}${SOABI_FLAG}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + venv/bin/pip install ${WHEEL_FILE} + + # Verify that the module is installed and peek at contents. + venv/bin/python scripts/check_crc32c_extension.py + unzip -l ${WHEEL_FILE} + + # Clean up. + rm -fr venv/ +done diff --git a/scripts/osx/build.sh b/scripts/osx/build.sh index bccfb838..56f88754 100755 --- a/scripts/osx/build.sh +++ b/scripts/osx/build.sh @@ -16,6 +16,9 @@ set -e -x echo "BUILDING FOR OSX" +# set deployment target +export MACOSX_DEPLOYMENT_TARGET=12 + # ``readlink -f`` is not our friend on OS X. This relies on **some** # ``python`` being installed. SCRIPT_FI=$(python -c "import os; print(os.path.realpath('${0}'))") @@ -23,47 +26,30 @@ OSX_DIR=$(dirname ${SCRIPT_FI}) SCRIPTS_DIR=$(dirname ${OSX_DIR}) export REPO_ROOT=$(dirname ${SCRIPTS_DIR}) -# NOTE: These are the Python.org versions of Python. -PYTHON37="/Library/Frameworks/Python.framework/Versions/3.7/bin" -PYTHON38="/Library/Frameworks/Python.framework/Versions/3.8/bin" -PYTHON39="/Library/Frameworks/Python.framework/Versions/3.9/bin" -PYTHON310="/Library/Frameworks/Python.framework/Versions/3.10/bin" -PYTHON311="/Library/Frameworks/Python.framework/Versions/3.11/bin" +# install required packages for pyenv +# https://github.com/pyenv/pyenv/wiki#suggested-build-environment +brew install openssl readline sqlite3 xz zlib tcl-tk # Build and install `libcrc32c` export PY_BIN="${PY_BIN:-python3}" export CRC32C_INSTALL_PREFIX="${REPO_ROOT}/usr" cd ${REPO_ROOT} +# Add directory as safe to avoid "detected dubious ownership" fatal issue +git config --global --add safe.directory $REPO_ROOT +git config --global --add safe.directory $REPO_ROOT/google_crc32c git submodule update --init --recursive ${OSX_DIR}/build_c_lib.sh -# Build wheel for Python 3.7. -export PY_BIN="python3.7" -export PY_TAG="cp37-cp37m" -${OSX_DIR}/build_python_wheel.sh - -# Build wheel for Python 3.8. -# Note that the 'm' SOABI flag is no longer supported for Python >= 3.8 -export PY_BIN="python3.8" -export PY_TAG="cp38-cp38" -${OSX_DIR}/build_python_wheel.sh - -# Build wheel for Python 3.9. -export PY_BIN="python3.9" -export PY_TAG="cp39-cp39" -${OSX_DIR}/build_python_wheel.sh - -# Build wheel for Python 3.10. -export PY_BIN="python3.10" -export PY_TAG="cp310-cp310" -${OSX_DIR}/build_python_wheel.sh +SUPPORTED_PYTHON_VERSIONS=("3.8" "3.9" "3.10" "3.11" "3.12") -# Build wheel for Python 3.11. -export PY_BIN="python3.11" -export PY_TAG="cp311-cp311" -${OSX_DIR}/build_python_wheel.sh +for PYTHON_VERSION in ${SUPPORTED_PYTHON_VERSIONS[@]}; do + echo "Build wheel for Python ${PYTHON_VERSION}" + export PY_BIN=$PYTHON_VERSION + export PY_TAG="cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}" + ${OSX_DIR}/build_python_wheel.sh +done # Clean up. rm -fr ${CRC32C_INSTALL_PREFIX} diff --git a/scripts/osx/build_c_lib.sh b/scripts/osx/build_c_lib.sh index 6a44322f..99b8bba7 100755 --- a/scripts/osx/build_c_lib.sh +++ b/scripts/osx/build_c_lib.sh @@ -54,23 +54,25 @@ ${VENV}/bin/python -m pip install "cmake >= 3.12.0" # Build `libcrc32c` cd ${REPO_ROOT}/google_crc32c mkdir -p build +cd build ls # We don't build i386 anymore as XCode no longer supports. ${VENV}/bin/cmake \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_OSX_DEPLOYMENT_TARGET=10.9 \ - -DCMAKE_OSX_ARCHITECTURES="x86_64" \ + -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" \ -DCRC32C_BUILD_TESTS=no \ -DCRC32C_BUILD_BENCHMARKS=no \ -DBUILD_SHARED_LIBS=yes \ -DCMAKE_INSTALL_PREFIX:PATH=${CRC32C_INSTALL_PREFIX} \ -DCMAKE_INSTALL_NAME_DIR:PATH=${CRC32C_INSTALL_PREFIX}/lib \ - . + .. # Install `libcrc32c` into CRC32C_INSTALL_PREFIX. make all install # Clean up. +cd .. rm -fr ${REPO_ROOT}/google_crc32c/build rm -fr ${VENV} diff --git a/scripts/osx/build_gh_action.sh b/scripts/osx/build_gh_action.sh deleted file mode 100755 index 8b3bc908..00000000 --- a/scripts/osx/build_gh_action.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -x -echo "BUILDING FOR OSX" - -# ``readlink -f`` is not our friend on OS X. This relies on **some** -# ``python`` being installed. -SCRIPT_FI=$(python -c "import os; print(os.path.realpath('${0}'))") -OSX_DIR=$(dirname ${SCRIPT_FI}) -SCRIPTS_DIR=$(dirname ${OSX_DIR}) -export REPO_ROOT=$(dirname ${SCRIPTS_DIR}) - -# Build and install `libcrc32c` -export PY_BIN="python3" -export CRC32C_INSTALL_PREFIX="${REPO_ROOT}/usr" - -cd ${REPO_ROOT} -git submodule update --init --recursive - -${OSX_DIR}/build_c_lib.sh - -# Build wheel -export PY_BIN="python3" -export PY_TAG="*" -${OSX_DIR}/build_python_wheel.sh - - -# Clean up. -rm -fr ${CRC32C_INSTALL_PREFIX} diff --git a/scripts/osx/build_python_wheel.sh b/scripts/osx/build_python_wheel.sh index 8865c771..30648992 100755 --- a/scripts/osx/build_python_wheel.sh +++ b/scripts/osx/build_python_wheel.sh @@ -33,9 +33,28 @@ if [[ -z "${PY_TAG}" ]]; then exit 1 fi +# set up pyenv & shell environment for switching across python versions +eval "$(pyenv init -)" +eval "$(pyenv init --path)" + +install_python_pyenv() { + version=$1 + + if [ -z "$(pyenv versions --bare | grep $version)" ]; then + echo "Python $version is not installed. Installing..." + pyenv install $version + echo "Python $version installed." + else + echo "Python $version is already installed." + fi + pyenv shell $version +} +install_python_pyenv ${PY_BIN} + # Create a virtualenv where we can install Python build dependencies. VENV=${REPO_ROOT}/venv${PY_BIN} -${PY_BIN} -m venv ${VENV} +"python${PY_BIN}" -m venv ${VENV} + curl https://bootstrap.pypa.io/get-pip.py | ${VENV}/bin/python ${VENV}/bin/python -m pip install \ --requirement ${REPO_ROOT}/scripts/dev-requirements.txt @@ -50,14 +69,23 @@ ${VENV}/bin/python setup.py build_ext \ --rpath=${REPO_ROOT}/usr/lib ${VENV}/bin/python -m pip wheel ${REPO_ROOT} --wheel-dir ${DIST_WHEELS} -# Delocate the wheel. removed --check-archs. We don't build i386. +# Delocate the wheel. FIXED_WHEELS="${REPO_ROOT}/wheels" mkdir -p ${FIXED_WHEELS} ${VENV}/bin/delocate-wheel \ --wheel-dir ${FIXED_WHEELS} \ --verbose \ + --check-archs \ ${DIST_WHEELS}/google_crc32c*${PY_TAG}*.whl +# Upload wheels to GCS for debugging. Uncomment only when needed. + +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} +ls ${REPO_ROOT}/wheels/ +gsutil cp ${REPO_ROOT}/wheels/* gs://python_crc32c/ +echo "Osx wheels uploaded successfully" + # Clean up. rm -fr ${DIST_WHEELS} rm -fr ${VENV} diff --git a/scripts/osx/check.sh b/scripts/osx/check.sh index 10d7b270..5d31c850 100755 --- a/scripts/osx/check.sh +++ b/scripts/osx/check.sh @@ -25,63 +25,39 @@ OSX_DIR=$(dirname ${SCRIPT_FI}) SCRIPTS_DIR=$(dirname ${OSX_DIR}) export REPO_ROOT=$(dirname ${SCRIPTS_DIR}) +# set up pyenv & shell environment for switching across python versions +eval "$(pyenv init -)" +eval "$(pyenv init --path)" ls ${REPO_ROOT}/wheels -# NOTE: These are the Python.org versions of Python. -PYTHON37="/Library/Frameworks/Python.framework/Versions/3.7/bin" -PYTHON38="/Library/Frameworks/Python.framework/Versions/3.8/bin" -PYTHON39="/Library/Frameworks/Python.framework/Versions/3.9/bin" -PYTHON310="/Library/Frameworks/Python.framework/Versions/3.10/bin" -PYTHON311="/Library/Frameworks/Python.framework/Versions/3.11/bin" - -# Make sure we have an updated `pip`. -curl https://bootstrap.pypa.io/get-pip.py | ${PYTHON37}/python3 - -# Make sure virtualenv and delocate. -${PYTHON37}/python3 -m pip install --upgrade delocate -LISTDEPS_CMD="${PYTHON37}/delocate-listdeps --all --depending" -VIRTUALENV_CMD="${PYTHON37}/python3 -m venv" - -${PYTHON37}/python3 -m venv venv37 -curl https://bootstrap.pypa.io/get-pip.py | venv37/bin/python3 -WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp37-cp37m-macosx_10_9_x86_64.whl -venv37/bin/pip install ${WHL} -venv37/bin/pip install pytest -venv37/bin/py.test ${REPO_ROOT}/tests -venv37/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py -${LISTDEPS_CMD} ${WHL} -rm -fr venv37 - -${PYTHON38}/python3 -m venv venv38 -curl https://bootstrap.pypa.io/get-pip.py | venv38/bin/python3 -WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp38-cp38-macosx_10_9_x86_64.whl -venv38/bin/pip install ${WHL} -venv38/bin/pip install pytest -venv38/bin/py.test ${REPO_ROOT}/tests -venv38/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py -${LISTDEPS_CMD} ${WHL} -rm -fr venv38 - -${PYTHON39}/python3 -m venv venv39 -curl https://bootstrap.pypa.io/get-pip.py | venv39/bin/python3 -WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp39-cp39-macosx_10_9_x86_64.whl -venv39/bin/pip install ${WHL} -venv39/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py -${LISTDEPS_CMD} ${WHL} -rm -fr venv39 - -${PYTHON310}/python3 -m venv venv310 -curl https://bootstrap.pypa.io/get-pip.py | venv310/bin/python3 -WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp310-cp310-macosx_10_9_x86_64.whl -venv310/bin/pip install ${WHL} -venv310/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py -${LISTDEPS_CMD} ${WHL} -rm -fr venv310 - -${PYTHON311}/python3 -m venv venv311 -curl https://bootstrap.pypa.io/get-pip.py | venv311/bin/python3 -WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp311-cp311-macosx_10_9_x86_64.whl -venv311/bin/pip install ${WHL} -venv311/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py -${LISTDEPS_CMD} ${WHL} -rm -fr venv311 +SUPPORTED_PYTHON_VERSIONS=("3.8" "3.9" "3.10" "3.11" "3.12") + +for PYTHON_VERSION in ${SUPPORTED_PYTHON_VERSIONS[@]}; do + PYTHON="python${PYTHON_VERSION}" + pyenv shell $PYTHON_VERSION + VIRTUALENV="venv${PYTHON_VERSION//.}" + # Make sure we have an updated `pip`. + curl https://bootstrap.pypa.io/get-pip.py | ${PYTHON} + # Make sure virtualenv and delocate. + ${PYTHON} -m pip install --upgrade delocate + LISTDEPS_CMD="${PYTHON}/delocate-listdeps --all --depending" + ${PYTHON} -m venv ${VIRTUALENV} + + OS_VERSION_STR="12_0" + if [ "${PYTHON_VERSION}" == "3.12" ]; then + OS_VERSION_STR="14.0" + fi + + #WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}-macosx_${OS_VERSION_STR}_x86_64.whl + #${VIRTUALENV}/bin/pip install ${WHL} --force-reinstall + + # Alternate method of finding the package that does not verify OS version is as expected + ${VIRTUALENV}/bin/pip install --no-index --find-links=${REPO_ROOT}/wheels google-crc32c --force-reinstall + + ${VIRTUALENV}/bin/pip install pytest + ${VIRTUALENV}/bin/py.test ${REPO_ROOT}/tests + ${VIRTUALENV}/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py + #${LISTDEPS_CMD} ${WHL} + rm -fr ${VIRTUALENV} + +done diff --git a/scripts/windows/build.bat b/scripts/windows/build.bat index d58b783c..a7b95ed9 100644 --- a/scripts/windows/build.bat +++ b/scripts/windows/build.bat @@ -15,40 +15,35 @@ setlocal ENABLEDELAYEDEXPANSION -set CMAKE_GENERATOR="Visual Studio 16 2019" +set CMAKE_GENERATOR="Visual Studio 17 2022" set CONFIGURATION=RelWithDebInfo set CRC32C_INSTALL_PREFIX=%cd%\build\%CONFIGURATION% -@rem Path to cmake, env var to make it easier to point to a specific version -set cmake=cmake +@rem Iterate through supported Python versions. +@rem Unfortunately pyenv for Windows has an out-of-date versions list. Choco's +@rem installer seems to have some problems with installing multiple versions at +@rem once, so as a workaround, we will install and then uninstall every version. +FOR %%P IN (3.8, 3.9, 3.10, 3.11, 3.12) DO ( -@rem python version should be set as an argument, if not, default to python 3.9 -set PYTHON_VERSION=%1 -if "%PYTHON_VERSION%"=="" ( - echo "Python version was not provided, using Python 3.9" - set PYTHON_VERSION=3.9 + echo "Installing Python version %%P" + choco install python --version=%%P -y --no-progress -) - -if "%PYTHON_VERSION%"=="3.10-dev" ( - set PYTHON_VERSION=3.10 -) + echo "Listing available Python versions' + py -0 -py -0 -py -%PYTHON_VERSION% -m pip install cmake + py -%%P-64 -m pip install --upgrade pip -git submodule update --init --recursive + echo "Installing cmake for Python %%P" + py -%%P-64 -m pip install cmake -FOR %%V IN (32,64) DO ( - set TARGET_PLATFORM="x64" - - if "%%V"=="32" ( - set TARGET_PLATFORM="Win32" - ) - echo "Target Platform: !TARGET_PLATFORM!" + @rem Add directory as safe to avoid "detected dubious ownership" fatal issue + git config --global --add safe.directory %cd% + git config --global --add safe.directory C:/tmpfs/src/github/python-crc32c + git submodule update --init --recursive + git config --global --add safe.directory %cd%\google_crc32c + git config --global --add safe.directory C:/tmpfs/src/github/python-crc32c/google_crc32c pushd google_crc32c - @rem reset hard to cleanup any changes done by a previous build. git reset --hard git clean -fxd @@ -59,11 +54,11 @@ FOR %%V IN (32,64) DO ( mkdir build cd build - echo "Running cmake with Generator: %CMAKE_GENERATOR%, Platform: !TARGET_PLATFORM!, Install Prefix: %CRC32C_INSTALL_PREFIX%" + echo "Running cmake with Generator: %CMAKE_GENERATOR%, Platform: x64, Install Prefix: %CRC32C_INSTALL_PREFIX%" - %cmake% -G %CMAKE_GENERATOR% -A !TARGET_PLATFORM! -DCMAKE_BUILD_TYPE=Release -DCRC32C_BUILD_BENCHMARKS=no -DCRC32C_BUILD_TESTS=no -DBUILD_SHARED_LIBS=yes -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=yes -DCRC32C_USE_GLOG=0 -DCMAKE_INSTALL_PREFIX:PATH=%CRC32C_INSTALL_PREFIX% .. + py -%%P-64 -m cmake -G %CMAKE_GENERATOR% -A x64 -DCRC32C_BUILD_BENCHMARKS=no -DCRC32C_BUILD_TESTS=no -DBUILD_SHARED_LIBS=yes -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=yes -DCRC32C_USE_GLOG=0 -DCMAKE_INSTALL_PREFIX:PATH=%CRC32C_INSTALL_PREFIX% .. - %cmake% --build . --config "%CONFIGURATION%" --target install + py -%%P-64 -m cmake --build . --config "%CONFIGURATION%" --target install dir %CRC32C_INSTALL_PREFIX% /b /s popd @@ -72,9 +67,26 @@ FOR %%V IN (32,64) DO ( echo "Copying Binary to root: %CRC32C_INSTALL_PREFIX%\bin\crc32c.dll" copy %CRC32C_INSTALL_PREFIX%\bin\crc32c.dll . - py -%PYTHON_VERSION%-%%V -m pip install --upgrade pip setuptools wheel + py -%%P-64 -m pip install --upgrade pip setuptools wheel echo "Building C extension" - py -%PYTHON_VERSION%-%%V setup.py build_ext --include-dirs=%CRC32C_INSTALL_PREFIX%\include --library-dirs=%CRC32C_INSTALL_PREFIX%\lib + py -%%P-64 setup.py build_ext -v --include-dirs=%CRC32C_INSTALL_PREFIX%\include --library-dirs=%CRC32C_INSTALL_PREFIX%\lib echo "Building Wheel" - py -%PYTHON_VERSION%-%%V -m pip wheel . --wheel-dir wheels/ + py -%%P-64 -m pip wheel . --wheel-dir wheels/ + + echo "Built wheel, now running tests." + call %~dp0/test.bat %%P + + echo "Finished with Python version %%P, now uninstalling" + choco uninstall python -y ) + +echo "Windows build has completed successfully" + +@REM Upload wheels to GCS for testing. Uncomment only when needed. + +set "GOOGLE_APPLICATION_CREDENTIALS=%KOKORO_GFILE_DIR%\service-account.json" +dir "%cd%\wheels" +call gcloud auth activate-service-account --key-file=%GOOGLE_APPLICATION_CREDENTIALS% +echo "Windows wheels uploading ..." +gsutil cp "%cd%\wheels\*" gs://python_crc32c/ +echo "Windows wheels uploaded successfully" diff --git a/scripts/windows/test.bat b/scripts/windows/test.bat index dc169a5c..5e9d1280 100644 --- a/scripts/windows/test.bat +++ b/scripts/windows/test.bat @@ -12,20 +12,18 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. -@rem python version should be set as an argument, if not, default to python 3.9 +@rem This test file runs for one Python version at a time, and is intended to +@rem be called from within the build loop. + set PYTHON_VERSION=%1 if "%PYTHON_VERSION%"=="" ( - echo "Python version was not provided, using Python 3.9" - set PYTHON_VERSION=3.9 + echo "Python version was not provided, using Python 3.10" + set PYTHON_VERSION=3.10 ) -@rem update python deps and build wheels (requires CRC32C_INSTALL_PREFIX is set) -@REM FOR %%V IN (3.9-64,3.9-32) DO ( -FOR %%V IN (%PYTHON_VERSION%-32, %PYTHON_VERSION%-64) DO ( - py -%%V -m pip install --no-index --find-links=wheels google-crc32c --force-reinstall +py -%PYTHON_VERSION%-64 -m pip install --no-index --find-links=wheels google-crc32c --force-reinstall - py -%%V ./scripts/check_crc32c_extension.py +py -%PYTHON_VERSION%-64 ./scripts/check_crc32c_extension.py - py -%%V -m pip install pytest - py -%%V -m pytest tests -) +py -%PYTHON_VERSION%-64 -m pip install pytest +py -%PYTHON_VERSION%-64 -m pytest tests diff --git a/setup.cfg b/setup.cfg index 65bc8dfe..aa2a1e31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ [metadata] name = google-crc32c -version = 1.5.0 +version = 1.6.0.dev1 description = A python wrapper of the C library 'Google CRC32C' url = https://github.com/googleapis/python-crc32c long_description = file: README.md diff --git a/setup.py b/setup.py index 4bb2aeca..5c304b20 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ def copy_dll(build_lib): - return # trying static build of C library. install_prefix = os.environ.get("CRC32C_INSTALL_PREFIX") if os.name == "nt" and install_prefix is not None: diff --git a/src/google_crc32c/__config__.py b/src/google_crc32c/__config__.py index ea220892..93b945bf 100644 --- a/src/google_crc32c/__config__.py +++ b/src/google_crc32c/__config__.py @@ -15,7 +15,6 @@ import os import sys - def modify_path(): """Modify the module search path.""" # Only modify path on Windows.