Merge branch 'main' into patch-1

.github/workflows/maturin.yml

@@ -0,0 +1,85 @@
+# This file is autogenerated by maturin v1.7.4 and manually modified by ia0.
+name: Maturin
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'python-v*'
+  schedule:
+    - cron: '12 3 * * 4' # Refresh the cache weekly.
+  workflow_dispatch:
+  # Uncomment to test the workflow in a PR.
+  # pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64
+          - runner: windows-latest
+            target: x64
+          - runner: macos-14
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v4
+      - if: matrix.platform.runner == 'ubuntu-latest'
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: rust/onnx/runtime/build/Linux
+          key: maturin-${{ matrix.platform.target }}-${{ hashFiles('rust/onnx/build.sh') }}
+      - run: python3 ./python/scripts/fix_package_version.py
+      - if: matrix.platform.runner == 'ubuntu-latest'
+        name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out=../dist
+          before-script-linux: "${{ github.workspace }}/rust/onnx/maturin.sh"
+          manylinux: 2_28
+          working-directory: python
+      - if: matrix.platform.runner != 'ubuntu-latest'
+        name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out=../dist
+          working-directory: python
+      - name: Install wheels
+        run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])")
+      - run: magika --version
+      - run: python3 -c 'import magika; print(magika.__version__)'
+      - run: magika -r tests_data/basic
+      - run: python3 ./python/scripts/run_quick_test_magika_cli.py
+      - run: python3 ./python/scripts/run_quick_test_magika_module.py
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # TODO(https://github.com/PyO3/maturin/issues/2244): Remove when released.
+      - run: rm rust/cli/README.md
+      - run: python3 ./python/scripts/fix_package_version.py
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out=../dist
+          working-directory: python
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-sdist
+          path: dist

python/pyproject.toml

@@ -74,8 +74,6 @@ module-name = "magika"
 bindings = "bin"
 manifest-path = "../rust/cli/Cargo.toml"
 locked = true
-compatibility = "linux"
-skip-auditwheel = true
 
 [tool.ruff.lint]
 # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.

python/scripts/fix_package_version.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# Copyright 2023-2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+
+# TODO(reyammer): Factorize this script into build_python_package.py.
+# TODO(https://github.com/PyO3/maturin/issues/2163): Remove this file when fixed.
+def main() -> None:
+    repo_root_dir = Path(__file__).parent.parent.parent
+    python_root_dir = repo_root_dir / "python"
+    rust_root_dir = repo_root_dir / "rust"
+    # Compute paths to files we'll need to restore at the end of the build
+    rust_main_rs_path = rust_root_dir / "cli" / "src" / "main.rs"
+    rust_cli_cargo_toml_path = rust_root_dir / "cli" / "Cargo.toml"
+
+    # get the rust version from Cargo.toml and patch main.rs
+    rust_version = get_rust_version(rust_root_dir)
+    patch_main_rs_with_version(rust_main_rs_path, rust_version)
+
+    # get the python version from magika.__version__ and patch Cargo.toml
+    python_version = get_python_version(python_root_dir)
+    patch_cargo_toml_with_version(rust_cli_cargo_toml_path, python_version)
+
+    # update Cargo.lock
+    subprocess.run(["cargo", "check"], cwd=rust_root_dir / "cli", check=True)
+
+
+def get_rust_version(rust_root_dir: Path) -> str:
+    cargo_path = rust_root_dir / "cli" / "Cargo.toml"
+    version = extract_with_regex(cargo_path, 'version = "([A-Za-z0-9.-]+)".*')
+    print(f"Extracted rust version: {version}")
+    return version
+
+
+def get_python_version(python_root_dir: Path) -> str:
+    init_path = python_root_dir / "src" / "magika" / "__init__.py"
+    version = extract_with_regex(init_path, '__version__ = "([A-Za-z0-9.-]+)"')
+    print(f"Extracted python version: {version}")
+    return version
+
+
+def patch_main_rs_with_version(rust_main_rs_path: Path, version: str) -> None:
+    print(f'Patching {rust_main_rs_path} with rust version "{version}"')
+    patch_line_matching_prefix(
+        rust_main_rs_path,
+        "        let binary = clap::crate_version!();",
+        f'        let binary = "{version}";',
+    )
+
+
+def patch_cargo_toml_with_version(cargo_toml_path: Path, version: str) -> None:
+    print(f'Patching {cargo_toml_path} with python version "{version}"')
+    patch_line_matching_prefix(cargo_toml_path, "version = ", f'version = "{version}"')
+
+
+def extract_with_regex(file_path: Path, regex: str) -> str:
+    """Extract a string via regex. This raises an exception if no or more than
+    one matches are found."""
+
+    lines = file_path.read_text().split("\n")
+    output = None
+    for line in lines:
+        m = re.fullmatch(regex, line)
+        if m:
+            if output is not None:
+                raise Exception(
+                    f'ERROR: Found more than one match for "{regex}" in {file_path}'
+                )
+            output = m.group(1)
+    if output is None:
+        raise Exception(f'No hits for "{regex}" in {file_path}')
+    return output
+
+
+def patch_line_matching_prefix(file_path: Path, prefix: str, new_line: str) -> None:
+    """Patch a line starting with a given prefix with a new line. This raises an
+    exception if no such line or more than one lines with a given prefix are
+    found."""
+
+    lines = file_path.read_text().split("\n")
+    already_found = False
+    for line_idx in range(len(lines)):
+        line = lines[line_idx]
+        if line.startswith(prefix):
+            if already_found:
+                raise Exception(
+                    f'ERROR: Found more than one line with prefix "{prefix}" in "{file_path}"'
+                )
+            already_found = True
+            lines[line_idx] = new_line
+
+    if not already_found:
+        raise Exception(
+            f'ERROR: Did not find any line with prefix "{prefix}" in "{file_path}"'
+        )
+    file_path.write_text("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()

rust/onnx/build.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+. ../color.sh
+
+# This script builds ONNX Runtime as a static library to be linked in the Magika CLI.
+#
+# This is needed when building for manylinux since the prebuilt binaries provided by the ort crate
+# have too recent dependency requirements.
+
+if [ -e runtime ]; then
+  info "Using cached static libraries."
+else
+  info "Make sure we have Python 3.x and cmake-3.27 or higher."
+  python3 -m venv venv
+  source venv/bin/activate
+  python3 -m pip install cmake
+
+  info "Clone ONNX Runtime repository (recursively)."
+  git clone --recursive https://github.com/Microsoft/onnxruntime.git runtime
+  cd runtime
+
+  info "Checkout v1.19.2 because that's what ort v2.0.0-rc.8 supports."
+  git checkout v1.19.2
+
+  info "Build the static libraries."
+  x ./build.sh --config=Release --parallel $ONNX_RUNTIME_BUILD_FLAGS
+
+  info "Only keep the static libraries to save cache space."
+  find build/Linux -not -name '*.a' \( -not -type d -or -empty \) -delete
+  cd ..
+fi
+
+info "Point the ort crate to the locally built static library."
+cd ../..
+cat >> .cargo/config.toml <<EOF
+
+[env]
+ORT_LIB_LOCATION = "$PWD/rust/onnx/runtime/build/Linux"
+EOF

rust/onnx/maturin.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+cd "$(dirname "$0")"
+. ../color.sh
+
+# This script sets up the maturin container for a manylinux build.
+
+info "Build ONNX Runtime from source."
+# We are root in maturin containers and ONNX Runtime doesn't like building as root by default.
+export ONNX_RUNTIME_BUILD_FLAGS=--allow_running_as_root
+./build.sh
+
+info "Test Magika CLI in the container."
+cd ../cli
+rustup default stable
+rustup component add rustfmt clippy
+./test.sh

tests_data/basic/rust/test_case1.rs

@@ -0,0 +1,12 @@
+/// Sample function to load a file
+fn load_model() {
+    println!("Magika model ⏳");
+    // Simulating a delay for loading
+    std::thread::sleep(std::time::Duration::from_millis(200));
+    println!("Model loaded successfully!");
+}
+
+fn main() {
+    // Load the model
+    load_model();
+}

tests_data/basic/rust/test_case2.rs

@@ -0,0 +1,28 @@
+use std::fs::File;
+use std::io::{Write, BufWriter}; 
+
+// Function to create a  CSV file
+fn create_csv() -> std::io::Result<()> {
+    let mut file = File::create("sample.csv")?;
+    writeln!(file, "Name,Age,City")?;
+    writeln!(file, "Alice,30,New York")?;
+    writeln!(file, "Bob,25,Los Angeles")?;
+    writeln!(file, "Charlie,35,Chicago")?;
+    Ok(())
+}
+
+// Function to create a  JSON file
+fn create_json() -> std::io::Result<()> {
+    let mut file = File::create("sample.json")?;
+    writeln!(file, "{{\"name\": \"Alice\", \"age\": 30, \"city\": \"New York\"}}")?;
+    Ok(())
+}
+
+// Main function 
+fn main() -> std::io::Result<()> {
+    // Function calls
+    create_csv()?;
+    create_json()?;
+    println!("Sample files created successfully.");
+    Ok(())
+}