From 14a4ffad973e5ab43f6da31144510e6a132a6652 Mon Sep 17 00:00:00 2001 From: ia0 Date: Sat, 2 Nov 2024 15:36:20 +0100 Subject: [PATCH] Test wheel in maturin container DO NOT MERGE. This is a test for #781. --- .github/workflows/codeql.yml | 87 ---------------------- .github/workflows/github-issue-labeler.yml | 20 ----- .github/workflows/github-pages.yml | 53 ------------- .github/workflows/js-docs-builder.yml | 40 ---------- .github/workflows/js-publish.yml | 29 -------- .github/workflows/js-test.yml | 39 ---------- .github/workflows/maturin.yml | 83 ++++++++++++--------- .github/workflows/python-test-suite.yml | 76 ------------------- .github/workflows/rust-test.yml | 47 ------------ .github/workflows/scorecard.yml | 70 ----------------- rust/test-wheel.sh | 31 ++++++++ 11 files changed, 77 insertions(+), 498 deletions(-) delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/github-issue-labeler.yml delete mode 100644 .github/workflows/github-pages.yml delete mode 100644 .github/workflows/js-docs-builder.yml delete mode 100644 .github/workflows/js-publish.yml delete mode 100644 .github/workflows/js-test.yml delete mode 100644 .github/workflows/python-test-suite.yml delete mode 100644 .github/workflows/rust-test.yml delete mode 100644 .github/workflows/scorecard.yml create mode 100755 rust/test-wheel.sh diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 66b39c33..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,87 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '42 13 * * 2' - -permissions: - contents: read - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} - permissions: - # required for all workflows - security-events: write - - # only required for workflows in private repositories - actions: read - contents: read - - strategy: - fail-fast: false - matrix: - language: [ 'javascript-typescript', 'python' ] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 - with: - category: "/language:${{matrix.language}}" diff --git a/.github/workflows/github-issue-labeler.yml b/.github/workflows/github-issue-labeler.yml deleted file mode 100644 index fe418ba8..00000000 --- a/.github/workflows/github-issue-labeler.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: New issue labeler -on: - # Runs on newly opened issues - issues: - types: [ opened ] - -# Sets permissions of the GITHUB_TOKEN -permissions: - issues: write - contents: read - -jobs: - triage: - runs-on: ubuntu-latest - steps: - - uses: github/issue-labeler@c1b0f9f52a63158c4adc09425e858e87b32e9685 # pin@v3.4 - with: - configuration-path: .github/labeler.yml - enable-versioned-regex: 0 - repo-token: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml deleted file mode 100644 index 66681e8a..00000000 --- a/.github/workflows/github-pages.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Pages - deploy - -on: - # Runs on pushes targeting the default branch - push: - branches: [ "main" ] - paths: [ website ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - - # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - deploy-pages: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./website - steps: - - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - with: - ref: main - - name: Set up Node - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # pin@v4 - with: - node-version: 18.x - - name: Install dependencies - run: npm ci - - name: Build - run: npm run build-github - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # pin@v4 - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # pin@v3 - with: - path: './website/dist' - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # pin@v4 diff --git a/.github/workflows/js-docs-builder.yml b/.github/workflows/js-docs-builder.yml deleted file mode 100644 index addd581f..00000000 --- a/.github/workflows/js-docs-builder.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: JS - generate docs - -on: - # Runs on pushes targeting the default branch - push: - branches: [ "main" ] - paths: - - 'js/**' - - '.github/workflows/**' - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -permissions: - contents: read - -jobs: - - makeDocs: - permissions: - contents: write - id-token: write - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - with: - ref: main - - name: Generate docs - working-directory: js - run: | - yarn - yarn run build - yarn run make-docs - - - name: Commit - run: | - git config --local user.email "invernizzi.l@gmail.com" - git config --local user.name "Luca Invernizzi" - git commit -m "Update docs" -a diff --git a/.github/workflows/js-publish.yml b/.github/workflows/js-publish.yml deleted file mode 100644 index 22e45412..00000000 --- a/.github/workflows/js-publish.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: JS - publish -on: - workflow_dispatch: - push: - tags: - - 'js-v[0-9]+.[0-9]+.[0-9]+' -permissions: - contents: read - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # pin@v4 - with: - node-version: '20.x' - registry-url: 'https://registry.npmjs.org' - - name: Build - working-directory: js - run: | - yarn - yarn run build - yarn run test - - name: Publish - working-directory: js - run: yarn publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/js-test.yml b/.github/workflows/js-test.yml deleted file mode 100644 index 0449ca9b..00000000 --- a/.github/workflows/js-test.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: JS - tests - -on: - workflow_dispatch: - push: - branches: - - 'main' - paths: - - 'js/**' - - 'tests_data/**' - - '.github/workflows/**' - pull_request: - paths: - - 'js/**' - - 'tests_data/**' - - '.github/workflows/**' -permissions: - contents: read -jobs: - unit-testing: - strategy: - matrix: - node-version: [ "18", "20" ] - os: [ "ubuntu-latest", "macos-latest" ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # pin@v4 - with: - node-version: ${{ matrix.node-version }} - - name: Install dependencies - working-directory: js - run: yarn - - name: Build - working-directory: js - run: yarn run build - - name: Run tests - working-directory: js - run: yarn test diff --git a/.github/workflows/maturin.yml b/.github/workflows/maturin.yml index fe2a3910..8c17335e 100644 --- a/.github/workflows/maturin.yml +++ b/.github/workflows/maturin.yml @@ -11,7 +11,7 @@ on: - cron: '12 3 * * 4' # Refresh the cache weekly. workflow_dispatch: # Uncomment to test the workflow in a PR. - # pull_request: + pull_request: permissions: contents: read @@ -24,10 +24,10 @@ jobs: platform: - runner: ubuntu-latest target: x86_64 - - runner: windows-latest - target: x64 - - runner: macos-14 - target: aarch64 + # - runner: windows-latest + # target: x64 + # - runner: macos-14 + # target: aarch64 steps: - uses: actions/checkout@v4 - if: matrix.platform.runner == 'ubuntu-latest' @@ -45,41 +45,50 @@ jobs: before-script-linux: "${{ github.workspace }}/rust/onnx/maturin.sh" manylinux: 2_28 working-directory: python - - if: matrix.platform.runner != 'ubuntu-latest' - name: Build wheels + - if: matrix.platform.runner == 'ubuntu-latest' + name: Test wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} args: --release --out=../dist + before-script-linux: "${{ github.workspace }}/rust/test-wheel.sh" + manylinux: 2_28 working-directory: python - - name: Install wheels - run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])") - - run: magika --version - - run: python3 -c 'import magika; print(magika.__version__)' - - run: magika -r tests_data/basic - - run: python3 ./python/scripts/run_quick_test_magika_cli.py - - run: python3 ./python/scripts/run_quick_test_magika_module.py - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} - path: dist + # - if: matrix.platform.runner != 'ubuntu-latest' + # name: Build wheels + # uses: PyO3/maturin-action@v1 + # with: + # target: ${{ matrix.platform.target }} + # args: --release --out=../dist + # working-directory: python + # - name: Install wheels + # run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])") + # - run: magika --version + # - run: python3 -c 'import magika; print(magika.__version__)' + # - run: magika -r tests_data/basic + # - run: python3 ./python/scripts/run_quick_test_magika_cli.py + # - run: python3 ./python/scripts/run_quick_test_magika_module.py + # - name: Upload wheels + # uses: actions/upload-artifact@v4 + # with: + # name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} + # path: dist - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - # TODO(https://github.com/PyO3/maturin/issues/2244): Remove when released. - - run: rm rust/cli/README.md - - run: python3 ./python/scripts/fix_package_version.py - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out=../dist - working-directory: python - - name: Upload sdist - uses: actions/upload-artifact@v4 - with: - name: wheels-sdist - path: dist + # sdist: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # # TODO(https://github.com/PyO3/maturin/issues/2244): Remove when released. + # - run: rm rust/cli/README.md + # - run: python3 ./python/scripts/fix_package_version.py + # - name: Build sdist + # uses: PyO3/maturin-action@v1 + # with: + # command: sdist + # args: --out=../dist + # working-directory: python + # - name: Upload sdist + # uses: actions/upload-artifact@v4 + # with: + # name: wheels-sdist + # path: dist diff --git a/.github/workflows/python-test-suite.yml b/.github/workflows/python-test-suite.yml deleted file mode 100644 index eca60af7..00000000 --- a/.github/workflows/python-test-suite.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Python - run test suite - -on: - workflow_dispatch: - push: - branches: - - 'main' - paths: - - 'python/**' - - 'tests_data/**' - - '.github/workflows/**' - pull_request: - paths: - - 'python/**' - - 'tests_data/**' - - '.github/workflows/**' - -permissions: - contents: read - -jobs: - unit-testing: - strategy: - matrix: - python-version: [ "3.8.x", "3.9.x", "3.10.x", "3.11.x", "3.12.x" ] - os: [ "ubuntu-latest", "macos-latest" ] - # TODO: add windows, ubuntu:20.04 - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - - - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # pin@v5 - with: - python-version: '${{ matrix.python-version }}' - - - name: Install uv - run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh - - - name: Install all projects dependencies (with the requested python version) - working-directory: python - run: uv sync --python $(which python3) --all-extras --dev - - - name: Run ruff check - working-directory: python - run: uv run --python $(which python3) ruff check --verbose - - - name: Run ruff format --check - working-directory: python - run: uv run --python $(which python3) ruff format --check --verbose - - - name: Run mypy - working-directory: python - run: uv run --python $(which python3) mypy src/magika tests - - - name: Run the python tests suite - working-directory: python - run: uv run --python $(which python3) pytest tests -m "not slow" - - - name: Run magika --version - working-directory: python - run: uv run magika --version - shell: bash # Allows for cross-platform - - - name: Run magika with tests_data - working-directory: python - # TODO(https://github.com/google/magika/issues/780): Remove "grep || exit 1" when fixed. - run: '(uv run magika -r ../tests_data/basic | grep "code\.asm.*Assembly") || exit 1' - - - name: Run "magika cli" quick tests - working-directory: python - run: uv run scripts/run_quick_test_magika_cli.py - - - name: Run "magika module" quick tests - working-directory: python - run: uv run scripts/run_quick_test_magika_module.py \ No newline at end of file diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml deleted file mode 100644 index a0476751..00000000 --- a/.github/workflows/rust-test.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Rust - test - -on: - workflow_dispatch: - push: - branches: - - 'main' - pull_request: - paths: - - '.github/workflows/**' - - 'assets/**' - - 'rust/**' - - 'tests_data/**' - -permissions: - contents: read - -jobs: - changelog: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - with: - fetch-depth: 0 - - run: ./changelog.sh - working-directory: rust - test: - runs-on: ubuntu-latest - strategy: - matrix: - toolchain: [stable, nightly] - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - - run: rustup default ${{ matrix.toolchain }} - - run: rustup component add rustfmt clippy - - run: ./test.sh - working-directory: rust - run: - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu, macos, windows] - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - - run: cargo build --release - working-directory: rust/cli - - run: rust/target/release/magika -r tests_data/basic diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml deleted file mode 100644 index d1c3a295..00000000 --- a/.github/workflows/scorecard.yml +++ /dev/null @@ -1,70 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - workflow_dispatch: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '45 21 * * 2' - push: - branches: [ "main" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - - steps: - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 - with: - sarif_file: results.sarif diff --git a/rust/test-wheel.sh b/rust/test-wheel.sh new file mode 100755 index 00000000..1192db28 --- /dev/null +++ b/rust/test-wheel.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +cd "$(dirname "$0")" +. ./color.sh + +# This script tests the wheel in the maturin container. + +cd .. +set -x +python3 -m ensurepip +python3 --version +python3 -m pip install dist/*.whl +magika --version +python3 -c 'import magika; print(magika.__version__)' +magika -r tests_data/basic +python3 ./python/scripts/run_quick_test_magika_cli.py +python3 ./python/scripts/run_quick_test_magika_module.py