diff --git a/.github/workflows/maturin.yml b/.github/workflows/maturin.yml new file mode 100644 index 00000000..a0206122 --- /dev/null +++ b/.github/workflows/maturin.yml @@ -0,0 +1,76 @@ +# This file is autogenerated by maturin v1.7.4 and manually modified by ia0. +name: CI + +on: + pull_request: + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + steps: + - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out=../dist + before-script-linux: "${{ github.workspace }}/rust/onnx/maturin.sh" + manylinux: 2_28 + working-directory: python + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist + + windows: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + steps: + - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out=../dist + working-directory: python + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: dist + + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-12 + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out=../dist + working-directory: python + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist diff --git a/python/pyproject.toml b/python/pyproject.toml index 8271663b..cfa4c67a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -74,8 +74,6 @@ module-name = "magika" bindings = "bin" manifest-path = "../rust/cli/Cargo.toml" locked = true -compatibility = "linux" -skip-auditwheel = true [tool.ruff.lint] # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. diff --git a/rust/cli/Cargo.lock b/rust/cli/Cargo.lock index ec65be50..2b16f9a0 100644 --- a/rust/cli/Cargo.lock +++ b/rust/cli/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "miniz_oxide 0.8.0", @@ -600,22 +600,19 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "ort" -version = "2.0.0-rc.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e45a172e6c0fb7d640e92c7740f4ea476bfc49ef5c52ea9c73e9fae32b09fe" +version = "2.0.0-rc.6" +source = "git+https://github.com/pykeio/ort.git#b19cff4de3197f18f2f3d662349a286791359bf0" dependencies = [ "half", "ndarray", "ort-sys", - "thiserror", "tracing", ] [[package]] name = "ort-sys" -version = "2.0.0-rc.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6fe264a9467cd0c19cbee07afe689fae9480c4706c4a1a00b5e64ff99ea83a" +version = "2.0.0-rc.6" +source = "git+https://github.com/pykeio/ort.git#b19cff4de3197f18f2f3d662349a286791359bf0" dependencies = [ "flate2", "pkg-config", @@ -742,9 +739,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -755,9 +752,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "log", "once_cell", @@ -770,15 +767,15 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -913,9 +910,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" dependencies = [ "filetime", "libc", @@ -993,21 +990,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" version = "0.1.32" @@ -1025,9 +1010,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -1037,9 +1022,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -1097,9 +1082,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] diff --git a/rust/cli/Cargo.toml b/rust/cli/Cargo.toml index 36e47858..6600dfcb 100644 --- a/rust/cli/Cargo.toml +++ b/rust/cli/Cargo.toml @@ -22,7 +22,11 @@ clap = { version = "4.5.9", features = ["cargo", "derive", "string"] } colored = "2.1.0" magika = { version = "=0.1.0-rc.1", path = "../lib", features = ["serde"] } num_cpus = "1.16.0" -ort = "2.0.0-rc.5" +ort = "2.0.0-rc.6" serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.120" tokio = { version = "1.38.1", features = ["full"] } + +# TODO: Temporary until 2.0.0-rc.7 is released. +[patch.crates-io.ort] +git = "https://github.com/pykeio/ort.git" diff --git a/rust/lib/Cargo.lock b/rust/lib/Cargo.lock index f517bee8..3723a07d 100644 --- a/rust/lib/Cargo.lock +++ b/rust/lib/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -356,21 +356,20 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "ort" -version = "2.0.0-rc.5" +version = "2.0.0-rc.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e45a172e6c0fb7d640e92c7740f4ea476bfc49ef5c52ea9c73e9fae32b09fe" +checksum = "a5f95fe501e1cb81dec2f66ee3129025759b602817aa2c77ff421390c418cc34" dependencies = [ "ndarray", "ort-sys", - "thiserror", "tracing", ] [[package]] name = "ort-sys" -version = "2.0.0-rc.5" +version = "2.0.0-rc.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6fe264a9467cd0c19cbee07afe689fae9480c4706c4a1a00b5e64ff99ea83a" +checksum = "b4174960a7b93a17564a05b26e05889f0dea9ee70e68db5841f27b40c0c9804e" dependencies = [ "flate2", "pkg-config", @@ -664,21 +663,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" version = "0.1.32" diff --git a/rust/lib/Cargo.toml b/rust/lib/Cargo.toml index 72919bd8..e5ea687c 100644 --- a/rust/lib/Cargo.toml +++ b/rust/lib/Cargo.toml @@ -27,7 +27,7 @@ thiserror = "1.0.63" tokio = { version = "1.38.1", features = ["fs", "io-util"] } [dependencies.ort] -version = "2.0.0-rc.5" +version = "2.0.0-rc.6" default-features = false features = ["ndarray"] diff --git a/rust/onnx/.gitignore b/rust/onnx/.gitignore new file mode 100644 index 00000000..a2ac47b6 --- /dev/null +++ b/rust/onnx/.gitignore @@ -0,0 +1 @@ +/runtime/ diff --git a/rust/onnx/build.sh b/rust/onnx/build.sh new file mode 100755 index 00000000..5d44d3f8 --- /dev/null +++ b/rust/onnx/build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +. ../color.sh + +# This script builds ONNX Runtime as a static library to be linked in the Magika CLI. +# +# This is needed when building for manylinux since the prebuilt binaries provided by the ort crate +# have too recent dependency requirements. + +[ -e runtime ] && error "This script can only be run once." + +info "Make sure we have Python 3.x and cmake-3.27 or higher." +python3 -m venv venv +source venv/bin/activate +python3 -m pip install cmake + +info "Clone ONNX Runtime repository (recursively)." +git clone --recursive https://github.com/Microsoft/onnxruntime.git runtime +cd runtime + +info "We checkout v1.19.2 because that's what ort v2.0.0-rc.7 supports." +git checkout v1.19.2 + +info "Build the static library." +./build.sh --config=Release --parallel $ONNX_RUNTIME_BUILD_FLAGS + +info "Point the ort crate to the locally built static library." +cd ../../.. +cat >> .cargo/config.toml <