From 1aa5e8513635f7df6bc09475c2e0fd16bae428f2 Mon Sep 17 00:00:00 2001 From: Konstantinos Paparas Date: Fri, 25 Oct 2024 13:53:41 +0200 Subject: [PATCH 1/6] chore: update cibw version and local scripts --- .github/workflows/build.yml | 2 +- scripts/linux/local-build.sh | 4 ++-- scripts/mac/local-build.sh | 4 ++-- scripts/win/local-build.ps1 | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7abd7ef..abf0aae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ on: env: LIB_VERSION: '2024.1.2' PYTHON_VERSION: '3.11.7' - CIBW_VERSION: '2.16.2' + CIBW_VERSION: '2.21.3' CIBW_BUILD: 'cp311-*' CIBW_SKIP: '*-musllinux_*' CIBW_BUILD_VERBOSITY: 1 diff --git a/scripts/linux/local-build.sh b/scripts/linux/local-build.sh index 6b81d72..94f9c3a 100755 --- a/scripts/linux/local-build.sh +++ b/scripts/linux/local-build.sh @@ -11,10 +11,10 @@ fi echo activating venv-ci source /tmp/.venv-ci/bin/activate -pip install cibuildwheel==2.13.1 +pip install cibuildwheel==2.21.3 export CIBW_BEFORE_BUILD='./build.sh' -export CIBW_BUILD='cp310-*' +export CIBW_BUILD='cp311-*' export CIBW_SKIP='*-musllinux_*' export CIBW_ARCHS='native' export CIBW_BUILD_VERBOSITY=1 diff --git a/scripts/mac/local-build.sh b/scripts/mac/local-build.sh index 577edb2..8ce09c3 100755 --- a/scripts/mac/local-build.sh +++ b/scripts/mac/local-build.sh @@ -11,10 +11,10 @@ fi echo activating venv-ci source /tmp/.venv-ci/bin/activate -pip3 install cibuildwheel==2.13.1 +pip3 install cibuildwheel==2.21.3 export CIBW_BEFORE_BUILD='./build.sh' -export CIBW_BUILD='cp310-*' +export CIBW_BUILD='cp311-*' export CIBW_SKIP='*-musllinux_*' export CIBW_ARCHS='native' export CIBW_BUILD_VERBOSITY=1 diff --git a/scripts/win/local-build.ps1 b/scripts/win/local-build.ps1 index cac830a..ec57f8d 100644 --- a/scripts/win/local-build.ps1 +++ b/scripts/win/local-build.ps1 @@ -20,11 +20,11 @@ if ((-not (Test-Path "$Env:Temp\.venv-ci" -PathType Container))) { echo "Activating .venv-ci" & $Env:Temp\.venv-ci\Scripts\activate.ps1 -pip install cibuildwheel==2.6.1 +pip install cibuildwheel==2.21.3 ExitOnFailure("Failed to activate rotki VirtualEnv") $env:CIBW_BEFORE_BUILD = 'PowerShell.exe -File .\build.ps1' -$env:CIBW_BUILD = 'cp39-*' +$env:CIBW_BUILD = 'cp311-*' $env:CIBW_ARCHS = 'native' $env:CIBW_BUILD_VERBOSITY = 1 From b25484eed9481df2510a43b5130c4fbb3137319c Mon Sep 17 00:00:00 2001 From: Konstantinos Paparas Date: Fri, 25 Oct 2024 15:05:10 +0200 Subject: [PATCH 2/6] feat: update openssl to v3 --- .github/workflows/build.yml | 4 ++-- openssl | 2 +- patches/patch-gen.py | 2 +- scripts/linux/build.sh | 19 +++++++++++-------- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index abf0aae..b54f8eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,11 +30,11 @@ jobs: ARCH: - NAME: x86_64 CFLAGS: "-mmacosx-version-min=10.09 -march=core2" - OPENSSLDIR: "/usr/local/etc/openssl@1.1" + OPENSSLDIR: "/usr/local/etc/openssl@3" FLAGS: no-shared no-asm no-idea no-camellia no-seed no-bf no-cast no-rc2 no-rc4 no-rc5 no-md2 no-md4 no-ecdh no-sock no-ssl3 no-dsa no-dh no-ec no-ecdsa no-tls1 no-rfc3779 no-whirlpool no-srp no-mdc2 no-ecdh no-engine no-srtp no-weak-ssl-ciphers - NAME: arm64 CFLAGS: "-mmacosx-version-min=11.0" - OPENSSLDIR: "/opt/homebrew/etc/openssl@1.1" + OPENSSLDIR: "/opt/homebrew/etc/openssl@3" FLAGS: no-shared no-asm no-idea no-camellia no-seed no-bf no-cast no-rc2 no-rc4 no-rc5 no-md2 no-md4 no-ecdh no-sock no-ssl3 no-dsa no-dh no-ec no-ecdsa no-tls1 no-rfc3779 no-whirlpool no-srp no-mdc2 no-ecdh no-engine no-srtp no-weak-ssl-ciphers name: "Build OpenSSL for macOS (${{ matrix.ARCH.NAME }})" steps: diff --git a/openssl b/openssl index e04bd34..c523121 160000 --- a/openssl +++ b/openssl @@ -1 +1 @@ -Subproject commit e04bd3433fd84e1861bf258ea37928d9845e6a86 +Subproject commit c523121f902fde2929909dc7f76b13ceb4961efe diff --git a/patches/patch-gen.py b/patches/patch-gen.py index 055b509..527cb79 100755 --- a/patches/patch-gen.py +++ b/patches/patch-gen.py @@ -17,7 +17,7 @@ "include": ".\\openssl\\include", }, "linux": { - "lib": "/usr/local/ssl/lib/", + "lib": "/usr/local/ssl/lib64/", "include": "/usr/local/ssl/include/", } } diff --git a/scripts/linux/build.sh b/scripts/linux/build.sh index f0a3049..3250d5f 100755 --- a/scripts/linux/build.sh +++ b/scripts/linux/build.sh @@ -5,7 +5,7 @@ WORKDIR=$PWD if [[ -n "$(command -v yum)" ]]; then echo "Installing tcl using yum" - yum -y install tcl-devel + yum -y install tcl-devel perl-IPC-Cmd fi if [[ -n "$(command -v apt-get)" ]]; then @@ -23,22 +23,25 @@ else exit 1 fi -echo "🏗️ Building OpenSSL" +echo "🏗 Building OpenSSL" cd openssl || exit 1 + +echo "⚙ Running Configure" ./Configure $OPENSSL_CONFIGURATION no-shared no-asm no-idea no-camellia no-weak-ssl-ciphers \ no-seed no-bf no-cast no-rc2 no-rc4 no-rc5 no-md2 \ no-md4 no-ecdh no-sock no-ssl3 \ no-dsa no-dh no-ec no-ecdsa no-tls1 \ no-rfc3779 no-whirlpool no-srp \ no-mdc2 no-ecdh no-engine no-srtp \ - --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null + --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null || exit 1 -make > /dev/null -make install_sw > /dev/null +echo "🔨 Running make" +make all > /dev/null || exit 1 +make install_sw > /dev/null || exit 1 -echo "✔️ OpenSSL Build Complete" +echo "✔ OpenSSL Build Complete" -echo "🏗️ Creating SQLCipher amalgamation" +echo "🏗 Creating SQLCipher amalgamation" cd "$WORKDIR/sqlcipher" || exit 1 @@ -50,7 +53,7 @@ cd "$WORKDIR/sqlcipher" || exit 1 make sqlite3.c > /dev/null -echo "✔️ SQLCipher amalgamation created" +echo "✔ SQLCipher amalgamation created" echo "Moving amalgamation to $WORKDIR/amalgamation" From 19a01a28b0c5cc5c457326113f5b78fcda556f06 Mon Sep 17 00:00:00 2001 From: Konstantinos Paparas Date: Fri, 25 Oct 2024 17:09:01 +0200 Subject: [PATCH 3/6] feat: extend wheel generation to include cp 3.12 --- .github/workflows/build.yml | 12 +++++++----- README.md | 2 +- scripts/linux/build.sh | 4 ++-- scripts/linux/local-build.sh | 4 ++-- scripts/mac/build.sh | 8 ++++---- scripts/mac/local-build.sh | 2 +- scripts/win/local-build.ps1 | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b54f8eb..0e37b6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,9 +12,9 @@ on: env: LIB_VERSION: '2024.1.2' - PYTHON_VERSION: '3.11.7' + PYTHON_VERSION: '3.11.9' CIBW_VERSION: '2.21.3' - CIBW_BUILD: 'cp311-*' + CIBW_BUILD: 'cp311-* cp312-*' CIBW_SKIP: '*-musllinux_*' CIBW_BUILD_VERBOSITY: 1 @@ -94,6 +94,7 @@ jobs: name: 'Build wheels for macOS' runs-on: macos-13 strategy: + fail-fast: false matrix: arch: [arm64, x86_64, universal2] needs: [build_macos_openssl_universal2] @@ -119,7 +120,7 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS_MACOS: ${{ matrix.arch }} - CIBW_BEFORE_BUILD_MACOS: ./build.sh + CIBW_BEFORE_ALL_MACOS: ./build.sh - uses: actions/upload-artifact@v4 with: name: pypi-wheels-macos-${{ matrix.arch }} @@ -129,6 +130,7 @@ jobs: name: 'Build wheels for Linux' runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: arch: [ aarch64, native ] steps: @@ -153,7 +155,7 @@ jobs: working-directory: build run: python -m cibuildwheel --output-dir wheelhouse env: - CIBW_BEFORE_BUILD_LINUX: ./build.sh + CIBW_BEFORE_ALL_LINUX: ./build.sh CIBW_ARCHS_LINUX: ${{ matrix.arch }} - uses: actions/upload-artifact@v4 with: @@ -180,7 +182,7 @@ jobs: working-directory: build run: python -m cibuildwheel --output-dir wheelhouse env: - CIBW_BEFORE_BUILD: Powershell.exe -F ./build.ps1 + CIBW_BEFORE_ALL_WINDOWS: Powershell.exe -F ./build.ps1 CIBW_ARCHS_WINDOWS: "native" - name: clean openssl conf shell: pwsh diff --git a/README.md b/README.md index 6a80f44..1f9e689 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![PyPI - Version](https://img.shields.io/pypi/v/rotki-pysqlcipher3) ![SQLCipher - Version](https://img.shields.io/badge/SQLCipher-v4.5.3-green) ![SQLite - Version](https://img.shields.io/badge/SQLite-3.39.4-green) -![OpenSSL - Version](https://img.shields.io/badge/OpenSSL-1.1.1w-green) +![OpenSSL - Version](https://img.shields.io/badge/OpenSSL-3.0.15-green) diff --git a/scripts/linux/build.sh b/scripts/linux/build.sh index 3250d5f..cb1fc72 100755 --- a/scripts/linux/build.sh +++ b/scripts/linux/build.sh @@ -49,9 +49,9 @@ cd "$WORKDIR/sqlcipher" || exit 1 --enable-tempstore=yes \ --disable-shared \ --enable-static=yes \ - --with-crypto-lib=none > /dev/null + --with-crypto-lib=none > /dev/null || exit 1 -make sqlite3.c > /dev/null +make sqlite3.c > /dev/null || exit 1 echo "✔ SQLCipher amalgamation created" diff --git a/scripts/linux/local-build.sh b/scripts/linux/local-build.sh index 94f9c3a..cf6ea4e 100755 --- a/scripts/linux/local-build.sh +++ b/scripts/linux/local-build.sh @@ -13,8 +13,8 @@ echo activating venv-ci source /tmp/.venv-ci/bin/activate pip install cibuildwheel==2.21.3 -export CIBW_BEFORE_BUILD='./build.sh' -export CIBW_BUILD='cp311-*' +export CIBW_BEFORE_ALL='./build.sh' +export CIBW_BUILD='cp311-* cp312-*' export CIBW_SKIP='*-musllinux_*' export CIBW_ARCHS='native' export CIBW_BUILD_VERBOSITY=1 diff --git a/scripts/mac/build.sh b/scripts/mac/build.sh index c6b9727..a392fcc 100755 --- a/scripts/mac/build.sh +++ b/scripts/mac/build.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash WORKDIR=$PWD -ARCH_POSTFIX=$(echo $_PYTHON_HOST_PLATFORM | sed -En 's/macosx-..\..-(.*)/\1/p') +ARCH_POSTFIX=${CIBW_ARCHS_MACOS} echo "Copying OpenSSL ${ARCH_POSTFIX} to openssl/" -cp -R "openssl-macos-$ARCH_POSTFIX" openssl/ +cp -R "openssl-macos-$ARCH_POSTFIX" openssl/ || exit 1 cd "sqlcipher" || exit 1 @@ -16,9 +16,9 @@ echo "Creating SQLCipher amalgamation" --enable-static=yes \ --with-crypto-lib=none \ CFLAGS="-DSQLITE_HAS_CODEC -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS -I$WORKDIR/openssl/include" \ - LDFLAGS="$WORKDIR/openssl/lib/libcrypto.a" > /dev/null + LDFLAGS="$WORKDIR/openssl/lib/libcrypto.a" > /dev/null || exit 1 -make sqlite3.c > /dev/null +make sqlite3.c > /dev/null || exit 1 if [[ ! -d "$WORKDIR/amalgamation" ]]; then mkdir -p "$WORKDIR/amalgamation" diff --git a/scripts/mac/local-build.sh b/scripts/mac/local-build.sh index 8ce09c3..0b2b25e 100755 --- a/scripts/mac/local-build.sh +++ b/scripts/mac/local-build.sh @@ -13,7 +13,7 @@ echo activating venv-ci source /tmp/.venv-ci/bin/activate pip3 install cibuildwheel==2.21.3 -export CIBW_BEFORE_BUILD='./build.sh' +export CIBW_BEFORE_ALL='./build.sh' export CIBW_BUILD='cp311-*' export CIBW_SKIP='*-musllinux_*' export CIBW_ARCHS='native' diff --git a/scripts/win/local-build.ps1 b/scripts/win/local-build.ps1 index ec57f8d..beb6d1c 100644 --- a/scripts/win/local-build.ps1 +++ b/scripts/win/local-build.ps1 @@ -23,7 +23,7 @@ echo "Activating .venv-ci" pip install cibuildwheel==2.21.3 ExitOnFailure("Failed to activate rotki VirtualEnv") -$env:CIBW_BEFORE_BUILD = 'PowerShell.exe -File .\build.ps1' +$env:CIBW_BEFORE_ALL = 'PowerShell.exe -File .\build.ps1' $env:CIBW_BUILD = 'cp311-*' $env:CIBW_ARCHS = 'native' $env:CIBW_BUILD_VERBOSITY = 1 From 03a9364da5ef056dc5bc7a9695ccc3136879645e Mon Sep 17 00:00:00 2001 From: Konstantinos Paparas Date: Fri, 25 Oct 2024 19:06:39 +0200 Subject: [PATCH 4/6] feat: update SQLcipher to v4.6.1 --- README.md | 8 ++++---- sqlcipher | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1f9e689..ef74a07 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # rotki-pysqlcipher3 ![PyPI - Version](https://img.shields.io/pypi/v/rotki-pysqlcipher3) -![SQLCipher - Version](https://img.shields.io/badge/SQLCipher-v4.5.3-green) -![SQLite - Version](https://img.shields.io/badge/SQLite-3.39.4-green) +![SQLCipher - Version](https://img.shields.io/badge/SQLCipher-v4.6.1-green) +![SQLite - Version](https://img.shields.io/badge/SQLite-3.46.1-green) ![OpenSSL - Version](https://img.shields.io/badge/OpenSSL-3.0.15-green) @@ -13,7 +13,7 @@ Configuration for providing pre-build [pysqlcipher3](https://github.com/rigglema This is a collections of patches and scripts to build wheels for rotki and publish them on PyPI. -It builds wheels for CPython 3.9-3.10 for the following architectures: +It builds wheels for CPython 3.11 & 3.12 for the following architectures: - Linux x86_64 - Linux aarch64 @@ -23,7 +23,7 @@ It builds wheels for CPython 3.9-3.10 for the following architectures: - macOS universal2 The package is intended to be a drop-in replacement for the [pysqlcipher3 package](https://pypi.org/project/pysqlcipher3/). -And it is statically linked with [SQLCipher](https://github.com/sqlcipher/sqlcipher) 4.x and [OpenSSL](https://github.com/openssl/openssl) 1.1.x. +And it is statically linked with [SQLCipher](https://github.com/sqlcipher/sqlcipher) 4.x and [OpenSSL](https://github.com/openssl/openssl) 3.0.x LTS. ## License The following license applies to the scripts and patches of this repo. diff --git a/sqlcipher b/sqlcipher index 8763afa..c5bd336 160000 --- a/sqlcipher +++ b/sqlcipher @@ -1 +1 @@ -Subproject commit 8763afaf13231cb1fc835b52c94ada23f8e47b3d +Subproject commit c5bd336ece77922433aaf6d6fe8cf203b0c299d5 From c697a7d13ed1510f683e09f7d100093ad240feb3 Mon Sep 17 00:00:00 2001 From: Konstantinos Paparas Date: Fri, 25 Oct 2024 19:24:57 +0200 Subject: [PATCH 5/6] fix: add conditional ssl path for linux x86_64/aarch64 --- .github/workflows/build.yml | 2 ++ patches/patch-gen.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0e37b6b..8be23f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,6 +149,8 @@ jobs: platforms: arm64 - name: prepare build directory run: ./scripts/prepare.sh + env: + CIBW_ARCHS_LINUX: ${{ matrix.arch }} - name: Install cibuildwheel run: python -m pip install cibuildwheel==${{ env.CIBW_VERSION }} - name: Build wheels diff --git a/patches/patch-gen.py b/patches/patch-gen.py index 527cb79..b7f91ed 100755 --- a/patches/patch-gen.py +++ b/patches/patch-gen.py @@ -1,12 +1,27 @@ #!/usr/bin/env python3 # -*- coding: ISO-8859-1 -*- +import os import sys +import platform from pathlib import Path import click import datetime from jinja2 import Template +linux_arch = os.getenv('CIBW_ARCHS_LINUX') + +if platform.system() == 'Linux': + if linux_arch is None or linux_arch == 'native': + linux_arch = platform.machine() + +# When running on x86_64 cibuildwheel will put it in lib64 but aarch64 will still use the lib directory +if linux_arch == 'x86_64': + openssl_lib_dir = "/usr/local/ssl/lib64/" +else: + openssl_lib_dir = "/usr/local/ssl/lib/" + + openssl = { "mac": { "lib": "./openssl/lib/", @@ -17,7 +32,7 @@ "include": ".\\openssl\\include", }, "linux": { - "lib": "/usr/local/ssl/lib64/", + "lib": openssl_lib_dir, "include": "/usr/local/ssl/include/", } } From bc59fac561da165cc2ca19c392170a3da3b4e5e2 Mon Sep 17 00:00:00 2001 From: Konstantinos Paparas Date: Fri, 25 Oct 2024 19:25:45 +0200 Subject: [PATCH 6/6] chore: change library version to 2024.10.1 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8be23f9..5172531 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ on: - 'v*' env: - LIB_VERSION: '2024.1.2' + LIB_VERSION: '2024.10.1' PYTHON_VERSION: '3.11.9' CIBW_VERSION: '2.21.3' CIBW_BUILD: 'cp311-* cp312-*'