diff --git a/.github/actions/build-push-npm-registry/action.yml b/.github/actions/build-napi-artifact/action.yml similarity index 62% rename from .github/actions/build-push-npm-registry/action.yml rename to .github/actions/build-napi-artifact/action.yml index faa7db2..9265171 100644 --- a/.github/actions/build-push-npm-registry/action.yml +++ b/.github/actions/build-napi-artifact/action.yml @@ -1,5 +1,5 @@ -name: "Build and Push packages to npm registry" -description: "Build and push an packages to npm registry" +name: "Build NAPI artifact" +description: "Build and upload artifact from rust build" inputs: runner_os: @@ -17,12 +17,16 @@ inputs: description: "Docker command that will run in the container" with_rust: description: "Should we setup Rust for this bin" - with_cuda: - description: "Should we setup cuda" + default: "true" node_auth_token: description: "node auth token" npm_token: description: "npm token" + shipyard_token: + description: "shipyard.rs token" + working-directory: + required: true + description: "directory of binary to build" bin: required: true description: "The binary to build" @@ -34,48 +38,45 @@ runs: id: read_package_json_version run: | VERSION=$(cat package.json | jq -r '.version' ) - echo "##[set-output name=current-version;]$(echo $VERSION)" + echo "current-version=$VERSION" >> $GITHUB_OUTPUT shell: bash - working-directory: crates/${{ inputs.bin }} + working-directory: ${{ inputs.working-directory }} + - name: Setup node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: 16 check-latest: true registry-url: https://npm.pkg.github.com/ scope: '@ForesightMiningSoftwareCorporation' + - name: Check if package with same version exists id: check-version-exists shell: bash - working-directory: crates/${{ inputs.bin }} + working-directory: ${{ inputs.working-directory }} run: | if npm view @foresightminingsoftwarecorporation/${{ inputs.bin }} --json > package_info.json ; then EXISTS=$(cat package_info.json | jq '.time | has("${{ steps.read_package_json_version.outputs.current-version }}")') else EXISTS="false" fi - echo "::set-output name=exists::$EXISTS" + echo "exists=$EXISTS" >> $GITHUB_OUTPUT + echo "home=$HOME" >> $GITHUB_OUTPUT env: NODE_AUTH_TOKEN: ${{ inputs.node_auth_token }} NPM_TOKEN: ${{ inputs.npm_token }} - name: Install rust toolchain uses: actions-rs/toolchain@v1 - if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' && inputs.with_rust == 'true' }} + if: ${{ steps.check-version-exists.outputs.exists == 'false' && contains(inputs.runner_os, 'windows') && inputs.with_rust == 'true' }} with: profile: minimal override: true toolchain: stable target: ${{ inputs.runner_arch }} - - name: Install CUDA toolkit - uses: Jimver/cuda-toolkit@v0.2.6 - id: cuda-toolkit - if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' && inputs.with_rust == 'true' && inputs.with_cuda == 'true' }} - with: - cuda: '11.5.0' - linux-local-args: '["--toolkit"]' - name: Build On Agent (native build) - if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' }} + if: ${{ steps.check-version-exists.outputs.exists == 'false' && contains(inputs.runner_os, 'windows') }} + env: + CARGO_HTTP_USER_AGENT: "shipyard ${{ inputs.shipyard_token }}" run: |- yarn install if [ "${{ inputs.with_rust }}" == "true" ]; then @@ -85,28 +86,32 @@ runs: fi yarn build $EXTRA_ARGS shell: bash - working-directory: crates/${{ inputs.bin }} + working-directory: ${{ inputs.working-directory }} - name: Build in docker uses: addnab/docker-run-action@v3 - if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os != 'windows-latest' }} + env: + CARGO_HTTP_USER_AGENT: "shipyard ${{ inputs.shipyard_token }}" + SSH_AUTH_SOCK: /ssh-agent + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + if: ${{ steps.check-version-exists.outputs.exists == 'false' && !contains(inputs.runner_os, 'windows') }} with: image: ${{ inputs.docker }} - options: --user 0:0 -w /build/crates/${{ inputs.bin }} -v ${{ github.workspace }}:/build -v ${{ env.SSH_AUTH_SOCK}}:/ssh-agent -v ${{ github.workspace }}/.github/workflows/gitconfig:/root/.gitconfig -e SSH_AUTH_SOCK=/ssh-agent -e CARGO_NET_GIT_FETCH_WITH_CLI=true + options: --user 0:0 -w /build/${{ inputs.working-directory }} -v ${{ github.workspace }}:/build -v ${{ env.SSH_AUTH_SOCK }}:/ssh-agent -v ${{ steps.check-version-exists.outputs.home }}/.gitconfig:/root/.gitconfig -v ${{ steps.check-version-exists.outputs.home }}/.ssh:/root/.ssh run: ${{ inputs.docker_build }} - name: Upload artifact - if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' }} - uses: actions/upload-artifact@v3 + if: ${{ steps.check-version-exists.outputs.exists == 'false' && contains(inputs.runner_os, 'windows') }} + uses: actions/upload-artifact@v4 with: name: bindings-${{ inputs.bin }}-x86_64-pc-windows-msvc - path: crates/${{ inputs.bin }}/${{ inputs.bin }}.*.node + path: ${{ inputs.working-directory }}/${{ inputs.bin }}.*.node if-no-files-found: error - name: Upload artifact - if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os != 'windows-latest' }} - uses: actions/upload-artifact@v3 + if: ${{ steps.check-version-exists.outputs.exists == 'false' && !contains(inputs.runner_os, 'windows') }} + uses: actions/upload-artifact@v4 with: name: bindings-${{ inputs.bin }}-x86_64-unknown-linux-musl - path: crates/${{ inputs.bin }}/${{ inputs.bin }}.*.node + path: ${{ inputs.working-directory }}/${{ inputs.bin }}.*.node if-no-files-found: error diff --git a/.github/workflows/build_napi_builder.yml b/.github/workflows/build_napi_builder.yml new file mode 100644 index 0000000..2865f41 --- /dev/null +++ b/.github/workflows/build_napi_builder.yml @@ -0,0 +1,29 @@ +name: Build builder for napi crates + +on: + workflow_dispatch: + push: + branches: napi + +jobs: + build-rust-test-image: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v3 + - name: Login to ACR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + # Build and Push Image with Cache + - name: Build & Push + uses: docker/build-push-action@v3 + with: + context: docker/napi_builder + push: true + tags: | + ghcr.io/foresightminingsoftwarecorporation/napi-rs-docker-builder:latest diff --git a/.github/workflows/npm-napi-release.yml b/.github/workflows/npm-napi-release.yml new file mode 100644 index 0000000..e77c3f4 --- /dev/null +++ b/.github/workflows/npm-napi-release.yml @@ -0,0 +1,136 @@ +on: + workflow_call: + inputs: + working-directory: + type: string + default: "" + +jobs: + check_release_npm: + name: Check NPM Packages + runs-on: "ubuntu-latest" + outputs: + name: ${{ steps.read_package_json.outputs.name }} + current-version: ${{ steps.read_package_json.outputs.current-version }} + exists: ${{ steps.check-version-exists.outputs.exists }} + steps: + - name: Git checkout + uses: actions/checkout@v3 + + - name: Get version in package.json + id: read_package_json + run: | + echo "name=$(cat package.json | jq -r '.name' | rev | cut -d'/' -f 1 | rev)" >> $GITHUB_OUTPUT + echo "current-version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT + working-directory: ${{ inputs.working-directory }} + + - name: Setup node + uses: actions/setup-node@v4 + with: + check-latest: true + registry-url: https://npm.pkg.github.com/ + scope: "@foresightminingsoftwarecorporation" + + - name: Check if package with same version exists + id: check-version-exists + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.GH_REGISTRY_PACKAGES }} + run: | + if npm view @foresightminingsoftwarecorporation/${{ steps.read_package_json.outputs.name }} --json > package_info.json ; then + EXISTS=$(cat package_info.json | jq '.time | has("${{ steps.read_package_json.outputs.current-version }}")') + else + EXISTS="false" + fi + echo "exists=$EXISTS" >> $GITHUB_OUTPUT + working-directory: ${{ inputs.working-directory }} + + build_artifacts_rust: + name: Build napi artifacts + needs: + - check_release_npm + if: ${{ needs.check_release_npm.outputs.exists == 'false' }} + strategy: + matrix: + os: + - name: ubuntu-latest + arch: x86_64-unknown-linux-gnu + ext: '' + - name: windows-latest + arch: x86_64-pc-windows-msvc + ext: .exe + runs-on: ${{ matrix.os.name }} + + steps: + - name: Git checkout + uses: actions/checkout@v4 + + - uses: ForesightMiningSoftwareCorporation/github/.github/actions/login-private-registry@v1 + with: + private_key: ${{ secrets.CARGO_PRIVATE_REGISTRY_SSH_PRIVATE_KEY }} + host: ${{ secrets.CARGO_PRIVATE_REGISTRY_HOST }} + name: ${{ secrets.CARGO_PRIVATE_REGISTRY_NAME }} + token: ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }} + additional_private_keys: | + ${{ secrets.FSE_SSH_PRIVATE_KEY }} + ${{ secrets.BEVY_CLIPMAP_SSH_PRIVATE_KEY }} + ${{ secrets.DAG_TABLES_SSH_PRIVATE_KEY }} + ${{ secrets.VOLUMESIGHT_SSH_PRIVATE_KEY }} + + - name: Build rust package + uses: ForesightMiningSoftwareCorporation/github/.github/actions/build-napi-artifact@napi + with: + runner_os: ${{ matrix.os.name }} + runner_arch: ${{ matrix.os.arch }} + runner_extension: ${{ matrix.os.ext }} + docker: ghcr.io/foresightminingsoftwarecorporation/napi-rs-docker-builder:latest + docker_build: ls -ahl ~/.ssh && yarn install && yarn build --target x86_64-unknown-linux-musl --js=false + node_auth_token: ${{ secrets.GITHUB_TOKEN }} + npm_token: ${{ secrets.GH_REGISTRY_PACKAGES }} + shipyard_token: ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }} + working-directory: ${{ inputs.working-directory }} + bin: ${{ needs.check_release_npm.outputs.name }} + + create_release_npm: + name: Publish NPM Packages + runs-on: "ubuntu-latest" + needs: + - check_release_npm + - build_artifacts_rust + steps: + - name: Install dependencies + if: ${{ needs.check_release_npm.outputs.exists == 'false' }} + run: yarn install + working-directory: ${{ inputs.working-directory }} + + - name: Download all artifacts + if: ${{ needs.check_release_npm.outputs.exists == 'false' }} + uses: actions/download-artifact@v3 + with: + path: ${{ inputs.working-directory }}/raw_artifacts + + - name: Move Artifacts and rename + if: ${{ needs.check_release_npm.outputs.exists == 'false' }} + shell: bash + run: |- + mkdir -p artifacts + for f in raw_artifacts/bindings-${{ needs.check_release_npm.outputs.name }}-*; do + NEW_NAME="${f/-${{ needs.check_release_npm.outputs.name }}/}" + mv -- "$f" "${NEW_NAME/raw_artifacts/artifacts}" + done + working-directory: ${{ inputs.working-directory }} + + - name: Move artifacts + if: ${{ needs.check_release_npm.outputs.exists == 'false' }} + run: yarn artifacts + working-directory: ${{ inputs.working-directory }} + + - name: Publish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.GH_REGISTRY_PACKAGES }} + if: ${{ false && needs.check_release_npm.outputs.exists == 'false' }} + run: | + npm publish + working-directory: ${{ inputs.working-directory }} diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 82473e0..5bbf436 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -491,7 +491,7 @@ jobs: const fmt = entry('cargo fmt --verbose --all -- --check', fetch_string('fmt_outcome', 'check'), true, 'fmt'); const check = entry(`cargo check ${ workspace_args } ${{ inputs.additional_args }}`, fetch_string('check_outcome', 'check'), true, 'check'); const clippy = entry(`cargo clippy ${ workspace_args } ${{ inputs.additional_args }} -- -D warnings`, fetch_string('clippy_outcome', 'check'), true, 'clippy'); - const doc = entry('cargo doc --no-deps', fetch_string('doc_outcome', 'check'), ${{ inputs.test-publish-required == 'true' }}, 'doc'); + const doc = entry('cargo doc --no-deps ${ workspace_args }', fetch_string('doc_outcome', 'check'), ${{ inputs.test-publish-required == 'true' }}, 'doc'); const licenses = entry('cargo deny check licenses', fetch_string('deny-license_outcome', 'check'), false, 'deny-license'); const bans = entry('cargo deny check bans', fetch_string('deny-bans_outcome', 'check'), false, 'deny-bans'); const advisories = entry('cargo deny check advisories', fetch_string('deny-advisories_outcome', 'check'), false, 'deny-advisories'); diff --git a/docker/napi_builder/Dockerfile b/docker/napi_builder/Dockerfile new file mode 100644 index 0000000..f39acbd --- /dev/null +++ b/docker/napi_builder/Dockerfile @@ -0,0 +1,33 @@ +FROM rust:alpine + +ENV PATH="/aarch64-linux-musl-cross/bin:/usr/local/cargo/bin/rustup:/root/.cargo/bin:$PATH" \ + RUSTFLAGS="-C target-feature=-crt-static" \ + CC="clang" \ + CXX="clang++" \ + GN_EXE=gn + +RUN apk add --update --no-cache nodejs yarn bash clang wget cmake git openssh make perl pkgconfig openssl-dev curl gcc musl-dev linux-headers + + +WORKDIR /tmp + +RUN curl -fLO "https://www.openssl.org/source/openssl-1.1.1m.tar.gz" +RUN tar xvzf "openssl-1.1.1m.tar.gz" +WORKDIR /tmp/openssl-1.1.1m +RUN ./config -fPIC +RUN make -j$(nproc) depend +RUN make -j$(nproc) +RUN make -j$(nproc) install_sw + +ENV OPENSSL_LIB_DIR=/usr/local/lib +ENV OPENSSL_INCLUDE_DIR=/usr/include/openssl +ENV OPENSSL_STATIC=yes + +WORKDIR /build + +RUN mkdir /ssh + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT /entrypoint.sh diff --git a/docker/napi_builder/entrypoint.sh b/docker/napi_builder/entrypoint.sh new file mode 100644 index 0000000..815995a --- /dev/null +++ b/docker/napi_builder/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +chown root:root ~/.gitconfig +chown -R root:root ~/.ssh + +sed 's|/home/runner|/root|g' -i.bak ~/.ssh/config +chmod -R 600 ~/.ssh + +# This will exec the CMD from your Dockerfile, i.e. "npm start" +exec "sh -c \"$@\""