Skip to content

Commit

Permalink
napi builder workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
greaka committed Jan 22, 2024
1 parent 44defde commit a26aafe
Show file tree
Hide file tree
Showing 6 changed files with 242 additions and 29 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "Build and Push packages to npm registry"
description: "Build and push an packages to npm registry"
name: "Build NAPI artifact"
description: "Build and upload artifact from rust build"

inputs:
runner_os:
Expand All @@ -17,12 +17,16 @@ inputs:
description: "Docker command that will run in the container"
with_rust:
description: "Should we setup Rust for this bin"
with_cuda:
description: "Should we setup cuda"
default: "true"
node_auth_token:
description: "node auth token"
npm_token:
description: "npm token"
shipyard_token:
description: "shipyard.rs token"
working-directory:
required: true
description: "directory of binary to build"
bin:
required: true
description: "The binary to build"
Expand All @@ -34,48 +38,45 @@ runs:
id: read_package_json_version
run: |
VERSION=$(cat package.json | jq -r '.version' )
echo "##[set-output name=current-version;]$(echo $VERSION)"
echo "current-version=$VERSION" >> $GITHUB_OUTPUT
shell: bash
working-directory: crates/${{ inputs.bin }}
working-directory: ${{ inputs.working-directory }}

- name: Setup node
uses: actions/setup-node@v3
uses: actions/setup-node@v4
with:
node-version: 16
check-latest: true
registry-url: https://npm.pkg.github.com/
scope: '@ForesightMiningSoftwareCorporation'

- name: Check if package with same version exists
id: check-version-exists
shell: bash
working-directory: crates/${{ inputs.bin }}
working-directory: ${{ inputs.working-directory }}
run: |
if npm view @foresightminingsoftwarecorporation/${{ inputs.bin }} --json > package_info.json ; then
EXISTS=$(cat package_info.json | jq '.time | has("${{ steps.read_package_json_version.outputs.current-version }}")')
else
EXISTS="false"
fi
echo "::set-output name=exists::$EXISTS"
echo "exists=$EXISTS" >> $GITHUB_OUTPUT
echo "home=$HOME" >> $GITHUB_OUTPUT
env:
NODE_AUTH_TOKEN: ${{ inputs.node_auth_token }}
NPM_TOKEN: ${{ inputs.npm_token }}
- name: Install rust toolchain
uses: actions-rs/toolchain@v1
if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' && inputs.with_rust == 'true' }}
if: ${{ steps.check-version-exists.outputs.exists == 'false' && contains(inputs.runner_os, 'windows') && inputs.with_rust == 'true' }}
with:
profile: minimal
override: true
toolchain: stable
target: ${{ inputs.runner_arch }}
- name: Install CUDA toolkit
uses: Jimver/cuda-toolkit@v0.2.6
id: cuda-toolkit
if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' && inputs.with_rust == 'true' && inputs.with_cuda == 'true' }}
with:
cuda: '11.5.0'
linux-local-args: '["--toolkit"]'

- name: Build On Agent (native build)
if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' }}
if: ${{ steps.check-version-exists.outputs.exists == 'false' && contains(inputs.runner_os, 'windows') }}
env:
CARGO_HTTP_USER_AGENT: "shipyard ${{ inputs.shipyard_token }}"
run: |-
yarn install
if [ "${{ inputs.with_rust }}" == "true" ]; then
Expand All @@ -85,28 +86,32 @@ runs:
fi
yarn build $EXTRA_ARGS
shell: bash
working-directory: crates/${{ inputs.bin }}
working-directory: ${{ inputs.working-directory }}

- name: Build in docker
uses: addnab/docker-run-action@v3
if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os != 'windows-latest' }}
env:
CARGO_HTTP_USER_AGENT: "shipyard ${{ inputs.shipyard_token }}"
SSH_AUTH_SOCK: /ssh-agent
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
if: ${{ steps.check-version-exists.outputs.exists == 'false' && !contains(inputs.runner_os, 'windows') }}
with:
image: ${{ inputs.docker }}
options: --user 0:0 -w /build/crates/${{ inputs.bin }} -v ${{ github.workspace }}:/build -v ${{ env.SSH_AUTH_SOCK}}:/ssh-agent -v ${{ github.workspace }}/.github/workflows/gitconfig:/root/.gitconfig -e SSH_AUTH_SOCK=/ssh-agent -e CARGO_NET_GIT_FETCH_WITH_CLI=true
options: --user 0:0 -w /build/${{ inputs.working-directory }} -v ${{ github.workspace }}:/build -v ${{ env.SSH_AUTH_SOCK }}:/ssh-agent -v ${{ steps.check-version-exists.outputs.home }}/.gitconfig:/root/.gitconfig -v ${{ steps.check-version-exists.outputs.home }}/.ssh:/root/.ssh
run: ${{ inputs.docker_build }}

- name: Upload artifact
if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os == 'windows-latest' }}
uses: actions/upload-artifact@v3
if: ${{ steps.check-version-exists.outputs.exists == 'false' && contains(inputs.runner_os, 'windows') }}
uses: actions/upload-artifact@v4
with:
name: bindings-${{ inputs.bin }}-x86_64-pc-windows-msvc
path: crates/${{ inputs.bin }}/${{ inputs.bin }}.*.node
path: ${{ inputs.working-directory }}/${{ inputs.bin }}.*.node
if-no-files-found: error

- name: Upload artifact
if: ${{ steps.check-version-exists.outputs.exists == 'false' && inputs.runner_os != 'windows-latest' }}
uses: actions/upload-artifact@v3
if: ${{ steps.check-version-exists.outputs.exists == 'false' && !contains(inputs.runner_os, 'windows') }}
uses: actions/upload-artifact@v4
with:
name: bindings-${{ inputs.bin }}-x86_64-unknown-linux-musl
path: crates/${{ inputs.bin }}/${{ inputs.bin }}.*.node
path: ${{ inputs.working-directory }}/${{ inputs.bin }}.*.node
if-no-files-found: error
29 changes: 29 additions & 0 deletions .github/workflows/build_napi_builder.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Build builder for napi crates

on:
workflow_dispatch:
push:
branches: napi

jobs:
build-rust-test-image:
runs-on: ubuntu-latest
steps:
- name: 'Checkout'
uses: actions/checkout@v3
- name: Login to ACR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
# Build and Push Image with Cache
- name: Build & Push
uses: docker/build-push-action@v3
with:
context: docker/napi_builder
push: true
tags: |
ghcr.io/foresightminingsoftwarecorporation/napi-rs-docker-builder:latest
136 changes: 136 additions & 0 deletions .github/workflows/npm-napi-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
on:
workflow_call:
inputs:
working-directory:
type: string
default: ""

jobs:
check_release_npm:
name: Check NPM Packages
runs-on: "ubuntu-latest"
outputs:
name: ${{ steps.read_package_json.outputs.name }}
current-version: ${{ steps.read_package_json.outputs.current-version }}
exists: ${{ steps.check-version-exists.outputs.exists }}
steps:
- name: Git checkout
uses: actions/checkout@v3

- name: Get version in package.json
id: read_package_json
run: |
echo "name=$(cat package.json | jq -r '.name' | rev | cut -d'/' -f 1 | rev)" >> $GITHUB_OUTPUT
echo "current-version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT
working-directory: ${{ inputs.working-directory }}

- name: Setup node
uses: actions/setup-node@v4
with:
check-latest: true
registry-url: https://npm.pkg.github.com/
scope: "@foresightminingsoftwarecorporation"

- name: Check if package with same version exists
id: check-version-exists
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.GH_REGISTRY_PACKAGES }}
run: |
if npm view @foresightminingsoftwarecorporation/${{ steps.read_package_json.outputs.name }} --json > package_info.json ; then
EXISTS=$(cat package_info.json | jq '.time | has("${{ steps.read_package_json.outputs.current-version }}")')
else
EXISTS="false"
fi
echo "exists=$EXISTS" >> $GITHUB_OUTPUT
working-directory: ${{ inputs.working-directory }}

build_artifacts_rust:
name: Build napi artifacts
needs:
- check_release_npm
if: ${{ needs.check_release_npm.outputs.exists == 'false' }}
strategy:
matrix:
os:
- name: ubuntu-latest
arch: x86_64-unknown-linux-gnu
ext: ''
- name: windows-latest
arch: x86_64-pc-windows-msvc
ext: .exe
runs-on: ${{ matrix.os.name }}

steps:
- name: Git checkout
uses: actions/checkout@v4

- uses: ForesightMiningSoftwareCorporation/github/.github/actions/login-private-registry@v1
with:
private_key: ${{ secrets.CARGO_PRIVATE_REGISTRY_SSH_PRIVATE_KEY }}
host: ${{ secrets.CARGO_PRIVATE_REGISTRY_HOST }}
name: ${{ secrets.CARGO_PRIVATE_REGISTRY_NAME }}
token: ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }}
additional_private_keys: |
${{ secrets.FSE_SSH_PRIVATE_KEY }}
${{ secrets.BEVY_CLIPMAP_SSH_PRIVATE_KEY }}
${{ secrets.DAG_TABLES_SSH_PRIVATE_KEY }}
${{ secrets.VOLUMESIGHT_SSH_PRIVATE_KEY }}
- name: Build rust package
uses: ForesightMiningSoftwareCorporation/github/.github/actions/build-napi-artifact@napi
with:
runner_os: ${{ matrix.os.name }}
runner_arch: ${{ matrix.os.arch }}
runner_extension: ${{ matrix.os.ext }}
docker: ghcr.io/foresightminingsoftwarecorporation/napi-rs-docker-builder:latest
docker_build: ls -ahl ~/.ssh && yarn install && yarn build --target x86_64-unknown-linux-musl --js=false
node_auth_token: ${{ secrets.GITHUB_TOKEN }}
npm_token: ${{ secrets.GH_REGISTRY_PACKAGES }}
shipyard_token: ${{ secrets.CARGO_PRIVATE_REGISTRY_TOKEN }}
working-directory: ${{ inputs.working-directory }}
bin: ${{ needs.check_release_npm.outputs.name }}

create_release_npm:
name: Publish NPM Packages
runs-on: "ubuntu-latest"
needs:
- check_release_npm
- build_artifacts_rust
steps:
- name: Install dependencies
if: ${{ needs.check_release_npm.outputs.exists == 'false' }}
run: yarn install
working-directory: ${{ inputs.working-directory }}

- name: Download all artifacts
if: ${{ needs.check_release_npm.outputs.exists == 'false' }}
uses: actions/download-artifact@v3
with:
path: ${{ inputs.working-directory }}/raw_artifacts

- name: Move Artifacts and rename
if: ${{ needs.check_release_npm.outputs.exists == 'false' }}
shell: bash
run: |-
mkdir -p artifacts
for f in raw_artifacts/bindings-${{ needs.check_release_npm.outputs.name }}-*; do
NEW_NAME="${f/-${{ needs.check_release_npm.outputs.name }}/}"
mv -- "$f" "${NEW_NAME/raw_artifacts/artifacts}"
done
working-directory: ${{ inputs.working-directory }}

- name: Move artifacts
if: ${{ needs.check_release_npm.outputs.exists == 'false' }}
run: yarn artifacts
working-directory: ${{ inputs.working-directory }}

- name: Publish
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.GH_REGISTRY_PACKAGES }}
if: ${{ false && needs.check_release_npm.outputs.exists == 'false' }}
run: |
npm publish
working-directory: ${{ inputs.working-directory }}
2 changes: 1 addition & 1 deletion .github/workflows/rust-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ jobs:
const fmt = entry('cargo fmt --verbose --all -- --check', fetch_string('fmt_outcome', 'check'), true, 'fmt');
const check = entry(`cargo check ${ workspace_args } ${{ inputs.additional_args }}`, fetch_string('check_outcome', 'check'), true, 'check');
const clippy = entry(`cargo clippy ${ workspace_args } ${{ inputs.additional_args }} -- -D warnings`, fetch_string('clippy_outcome', 'check'), true, 'clippy');
const doc = entry('cargo doc --no-deps', fetch_string('doc_outcome', 'check'), ${{ inputs.test-publish-required == 'true' }}, 'doc');
const doc = entry('cargo doc --no-deps ${ workspace_args }', fetch_string('doc_outcome', 'check'), ${{ inputs.test-publish-required == 'true' }}, 'doc');
const licenses = entry('cargo deny check licenses', fetch_string('deny-license_outcome', 'check'), false, 'deny-license');
const bans = entry('cargo deny check bans', fetch_string('deny-bans_outcome', 'check'), false, 'deny-bans');
const advisories = entry('cargo deny check advisories', fetch_string('deny-advisories_outcome', 'check'), false, 'deny-advisories');
Expand Down
33 changes: 33 additions & 0 deletions docker/napi_builder/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
FROM rust:alpine

ENV PATH="/aarch64-linux-musl-cross/bin:/usr/local/cargo/bin/rustup:/root/.cargo/bin:$PATH" \
RUSTFLAGS="-C target-feature=-crt-static" \
CC="clang" \
CXX="clang++" \
GN_EXE=gn

RUN apk add --update --no-cache nodejs yarn bash clang wget cmake git openssh make perl pkgconfig openssl-dev curl gcc musl-dev linux-headers


WORKDIR /tmp

RUN curl -fLO "https://www.openssl.org/source/openssl-1.1.1m.tar.gz"
RUN tar xvzf "openssl-1.1.1m.tar.gz"
WORKDIR /tmp/openssl-1.1.1m
RUN ./config -fPIC
RUN make -j$(nproc) depend
RUN make -j$(nproc)
RUN make -j$(nproc) install_sw

ENV OPENSSL_LIB_DIR=/usr/local/lib
ENV OPENSSL_INCLUDE_DIR=/usr/include/openssl
ENV OPENSSL_STATIC=yes

WORKDIR /build

RUN mkdir /ssh

COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh

ENTRYPOINT /entrypoint.sh
10 changes: 10 additions & 0 deletions docker/napi_builder/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/sh

chown root:root ~/.gitconfig
chown -R root:root ~/.ssh

sed 's|/home/runner|/root|g' -i.bak ~/.ssh/config
chmod -R 600 ~/.ssh

# This will exec the CMD from your Dockerfile, i.e. "npm start"
exec "sh -c \"$@\""

0 comments on commit a26aafe

Please sign in to comment.