From d02bbf96d8ba24a05c1b26fdb46437cb23af10ea Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 04:40:58 +0800 Subject: [PATCH 01/10] build for cp310 --- .github/workflows/release_python.yml | 27 ++++++++++++++++++++++++--- bindings/python/Cargo.toml | 4 +++- bindings/python/pyproject.toml | 3 +-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index be849b8327db..4441c09280c3 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -66,7 +66,17 @@ jobs: manylinux: auto working-directory: "bindings/python" command: build - args: --release -o dist --find-interpreter --features=pyo3/extension-module,services-all + args: --release -o dist --find-interpreter -i python3.11 --features=pyo3/extension-module,services-all,abi3 + env: + # Workaround ring 0.17 build issue + CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" + - uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: auto + working-directory: "bindings/python" + command: build + args: --release -o dist --find-interpreter -i python3.10 --features=pyo3/extension-module,services-all env: # Workaround ring 0.17 build issue CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" @@ -86,7 +96,12 @@ jobs: with: working-directory: "bindings/python" command: build - args: --release -o dist --find-interpreter --features=pyo3/extension-module,services-all + args: --release -o dist --find-interpreter -i python3.11 --features=pyo3/extension-module,services-all,abi3 + - uses: PyO3/maturin-action@v1 + with: + working-directory: "bindings/python" + command: build + args: --release -o dist --find-interpreter -i python3.10 --features=pyo3/extension-module,services-all - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -104,7 +119,13 @@ jobs: working-directory: "bindings/python" command: build target: universal2-apple-darwin - args: --release -o dist --find-interpreter --features=pyo3/extension-module,services-all + args: --release -o dist --find-interpreter -i python3.11 --features=pyo3/extension-module,services-all,abi3 + - uses: PyO3/maturin-action@v1 + with: + working-directory: "bindings/python" + command: build + target: universal2-apple-darwin + args: --release -o dist --find-interpreter -i python3.10 --features=pyo3/extension-module,services-all - name: Upload wheels uses: actions/upload-artifact@v3 with: diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 9f418bd9a2dc..34c52b7d2cbd 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -150,6 +150,8 @@ services-upyun = ["opendal/services-upyun"] services-vercel-artifacts = ["opendal/services-vercel-artifacts"] services-yandex-disk = ["opendal/services-yandex-disk"] +abi3 = ["pyo3/abi3-py311"] + [lib] crate-type = ["cdylib"] doc = false @@ -161,7 +163,7 @@ futures = "0.3.28" opendal = { version = ">=0", path = "../../core", features = [ "layers-blocking", ] } -pyo3 = { version = "0.22.5", features = ["abi3", "abi3-py311"] } +pyo3 = { version = "0.22.5", features = [] } pyo3-async-runtimes = { version = "0.22.0", features = [ "tokio-runtime", ] } diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index fd58e48ff34a..f718f33b0dff 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -23,13 +23,12 @@ requires = ["maturin>=1.0,<2.0"] classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", ] description = "Apache OpenDAL™ Python Binding" license = { text = "Apache-2.0" } name = "opendal" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.10" [project.optional-dependencies] benchmark = [ From a073c2e5469c098bf5d9194fd98ad178b864e92f Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 04:43:18 +0800 Subject: [PATCH 02/10] fix --- .github/workflows/release_python.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 4441c09280c3..cad09bc85a8f 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -66,7 +66,7 @@ jobs: manylinux: auto working-directory: "bindings/python" command: build - args: --release -o dist --find-interpreter -i python3.11 --features=pyo3/extension-module,services-all,abi3 + args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 env: # Workaround ring 0.17 build issue CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" @@ -76,7 +76,7 @@ jobs: manylinux: auto working-directory: "bindings/python" command: build - args: --release -o dist --find-interpreter -i python3.10 --features=pyo3/extension-module,services-all + args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all env: # Workaround ring 0.17 build issue CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" @@ -96,12 +96,12 @@ jobs: with: working-directory: "bindings/python" command: build - args: --release -o dist --find-interpreter -i python3.11 --features=pyo3/extension-module,services-all,abi3 + args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" command: build - args: --release -o dist --find-interpreter -i python3.10 --features=pyo3/extension-module,services-all + args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -119,13 +119,13 @@ jobs: working-directory: "bindings/python" command: build target: universal2-apple-darwin - args: --release -o dist --find-interpreter -i python3.11 --features=pyo3/extension-module,services-all,abi3 + args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" command: build target: universal2-apple-darwin - args: --release -o dist --find-interpreter -i python3.10 --features=pyo3/extension-module,services-all + args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all - name: Upload wheels uses: actions/upload-artifact@v3 with: From a43a0489be54e19959ec8c6d96debdac38af4087 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 04:48:09 +0800 Subject: [PATCH 03/10] sccache --- .github/workflows/release_python.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index cad09bc85a8f..77f0393b041e 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -67,6 +67,7 @@ jobs: working-directory: "bindings/python" command: build args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 + sccache: true env: # Workaround ring 0.17 build issue CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" @@ -77,6 +78,7 @@ jobs: working-directory: "bindings/python" command: build args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all + sccache: true env: # Workaround ring 0.17 build issue CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" @@ -97,11 +99,13 @@ jobs: working-directory: "bindings/python" command: build args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 + sccache: true - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" command: build args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all + sccache: true - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -120,12 +124,14 @@ jobs: command: build target: universal2-apple-darwin args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 + sccache: true - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" command: build target: universal2-apple-darwin args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all + sccache: true - name: Upload wheels uses: actions/upload-artifact@v3 with: From d43274f662b7e9d745680084238272a7aad1bd6c Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 05:16:20 +0800 Subject: [PATCH 04/10] avoid compile local crate twice --- .github/workflows/release_python.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 77f0393b041e..f95b341e9ec9 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -90,6 +90,9 @@ jobs: windows: runs-on: windows-latest + env: + # help avoid compile local crate twice + CARGO_TARGET_DIR: './target/' steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain @@ -114,6 +117,9 @@ jobs: macos: runs-on: macos-latest + env: + # help avoid compile local crate twice + CARGO_TARGET_DIR: './target/' steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain From d8fc9737fa9148aeaa3ad85e59955235ff9d3422 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 05:21:33 +0800 Subject: [PATCH 05/10] use matrix to avoid write anything twice --- .github/workflows/release_python.yml | 84 +++++----------------------- 1 file changed, 15 insertions(+), 69 deletions(-) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index f95b341e9ec9..9dad16ac923b 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -20,7 +20,7 @@ name: Release Python on: push: tags: - - '*' + - "*" pull_request: branches: - main @@ -51,75 +51,21 @@ jobs: name: wheels path: bindings/python/dist - linux: - runs-on: ubuntu-latest + wheels: + runs-on: "${{ matrix.os }}" strategy: matrix: - target: [ x86_64, aarch64, armv7l ] - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - manylinux: auto - working-directory: "bindings/python" - command: build - args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 - sccache: true - env: - # Workaround ring 0.17 build issue - CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" - - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - manylinux: auto - working-directory: "bindings/python" - command: build - args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all - sccache: true - env: - # Workaround ring 0.17 build issue - CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: bindings/python/dist - - windows: - runs-on: windows-latest - env: - # help avoid compile local crate twice - CARGO_TARGET_DIR: './target/' - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - uses: PyO3/maturin-action@v1 - with: - working-directory: "bindings/python" - command: build - args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 - sccache: true - - uses: PyO3/maturin-action@v1 - with: - working-directory: "bindings/python" - command: build - args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all - sccache: true - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: bindings/python/dist - - macos: - runs-on: macos-latest + include: + - { os: windows-latest } + - { os: macos-latest, target: "universal2-apple-darwin" } + - { os: ubuntu-latest, target: "x86_64" } + - { os: ubuntu-latest, target: "aarch64" } + - { os: ubuntu-latest, target: "armv7l" } env: # help avoid compile local crate twice - CARGO_TARGET_DIR: './target/' + CARGO_TARGET_DIR: "./target/" + # Workaround ring 0.17 build issue + CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain @@ -127,15 +73,15 @@ jobs: - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" + target: "${{ matrix.target }}" command: build - target: universal2-apple-darwin args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 sccache: true - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" + target: "${{ matrix.target }}" command: build - target: universal2-apple-darwin args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all sccache: true - name: Upload wheels @@ -151,7 +97,7 @@ jobs: permissions: contents: read id-token: write - needs: [ macos, linux, windows ] + needs: [sdist, wheels] steps: - uses: actions/download-artifact@v3 with: From 1ed0c8ef43297faedbc59a59ff3ac6629fac95d2 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 05:42:47 +0800 Subject: [PATCH 06/10] speedup doesn't really work --- .github/workflows/release_python.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 9dad16ac923b..971c882fdaa0 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -62,8 +62,6 @@ jobs: - { os: ubuntu-latest, target: "aarch64" } - { os: ubuntu-latest, target: "armv7l" } env: - # help avoid compile local crate twice - CARGO_TARGET_DIR: "./target/" # Workaround ring 0.17 build issue CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" steps: From abf3b96e50da7ef89fae2b6117e36b8e1ee3a5b3 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 08:09:49 +0800 Subject: [PATCH 07/10] Update release_python.yml --- .github/workflows/release_python.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 971c882fdaa0..c00803e5d3cf 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -75,6 +75,7 @@ jobs: command: build args: --release -o dist -i python3.11 --features=pyo3/extension-module,services-all,abi3 sccache: true + manylinux: auto - uses: PyO3/maturin-action@v1 with: working-directory: "bindings/python" @@ -82,6 +83,7 @@ jobs: command: build args: --release -o dist -i python3.10 --features=pyo3/extension-module,services-all sccache: true + manylinux: auto - name: Upload wheels uses: actions/upload-artifact@v3 with: From 012a66ec47a8c1449c08fa58c693deeee5f880fe Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 10:38:38 +0800 Subject: [PATCH 08/10] Update bindings/python/Cargo.toml Co-authored-by: Xuanwo --- bindings/python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 34c52b7d2cbd..fab77738910d 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -163,7 +163,7 @@ futures = "0.3.28" opendal = { version = ">=0", path = "../../core", features = [ "layers-blocking", ] } -pyo3 = { version = "0.22.5", features = [] } +pyo3 = "0.22.5" pyo3-async-runtimes = { version = "0.22.0", features = [ "tokio-runtime", ] } From 59a568d9f7daa7c6185c0ecd7df4c50ed653de57 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 10:40:59 +0800 Subject: [PATCH 09/10] Update bindings/python/Cargo.toml --- bindings/python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index fab77738910d..b2468dd8070d 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -149,7 +149,7 @@ services-tikv = ["opendal/services-tikv"] services-upyun = ["opendal/services-upyun"] services-vercel-artifacts = ["opendal/services-vercel-artifacts"] services-yandex-disk = ["opendal/services-yandex-disk"] - +# we build cp311-abi3 and cp310 wheels now, mive this to pyo3 after we drop cp319 abi3 = ["pyo3/abi3-py311"] [lib] From 9f1e81cf0c4e10d7c0412ebf2cd4e4c6664be950 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 20 Nov 2024 10:41:34 +0800 Subject: [PATCH 10/10] Apply suggestions from code review --- bindings/python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index b2468dd8070d..bee110942267 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -149,7 +149,7 @@ services-tikv = ["opendal/services-tikv"] services-upyun = ["opendal/services-upyun"] services-vercel-artifacts = ["opendal/services-vercel-artifacts"] services-yandex-disk = ["opendal/services-yandex-disk"] -# we build cp311-abi3 and cp310 wheels now, mive this to pyo3 after we drop cp319 +# we build cp311-abi3 and cp310 wheels now, move this to pyo3 after we drop cp310 abi3 = ["pyo3/abi3-py311"] [lib]