jwst.yml

name: Build Jwst

on:
    workflow_dispatch:
    push:
        branches: [master]
        paths:
            - 'apps/cloud/**'
            - 'apps/homepage/**'
            - 'apps/keck/**'
            - 'libs/**'
            - '.github/deployment/Dockerfile-cloud'
            - '.github/deployment/Dockerfile-jwst'
            - '.github/deployment/Dockerfile-mt-cloud'
            - '.github/workflows/jwst.yml'
    pull_request:
        branches: [master]
        paths:
            - 'apps/cloud/**'
            - 'apps/homepage/**'
            - 'apps/playground/**'
            - 'apps/keck/**'
            - 'libs/**'
            - '.github/deployment/Dockerfile-cloud'
            - '.github/deployment/Dockerfile-jwst'
            - '.github/deployment/Dockerfile-mt-cloud'
            - '.github/workflows/jwst.yml'

# Cancels all previous workflow runs for pull requests that have not completed.
# See https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
    # The concurrency group contains the workflow name and the branch name for
    # pull requests or the commit hash for any other events.
    group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
    cancel-in-progress: true

env:
    REGISTRY: ghcr.io
    NAMESPACE: toeverything

    APIPROXY_IMAGE_NAME: apiproxy
    CLOUD_IMAGE_NAME: cloud-self-hosted
    JWST_IMAGE_NAME: jwst

    IMAGE_TAG: canary-${{ github.sha }}
    IMAGE_TAG_LATEST: nightly-latest

    LOCAL_CACHE: localhost:5000/toeverything/relocate:latest
    IMAGE_TAG_MT: canary-mt-${{ github.sha }}
    IMAGE_TAG_MT_LATEST: mt-latest

jobs:
    aio-cloud-baseline:
        if: github.ref == 'refs/heads/master'
        runs-on: [self-hosted, linux, x64]
        environment: development
        permissions:
            contents: read
            packages: write

        services:
            registry:
                image: registry:2
                ports:
                    - 5000:5000

        steps:
            - name: Checkout
              uses: actions/checkout@v2

            - name: Install Rust
              uses: actions-rs/toolchain@v1
              with:
                  profile: minimal
                  toolchain: stable
                  override: true

            - uses: Swatinem/rust-cache@v2
            - name: Set up pnpm cache
              uses: actions/cache@v3
              continue-on-error: false
              with:
                  path: ./node_modules/.pnpm-store
                  key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
                  restore-keys: ${{ runner.os }}-pnpm-

            - name: Cargo Vendor
              run: cargo vendor > .cargo/config

            - name: Log in to the Container registry
              uses: docker/login-action@v2
              if: github.ref == 'refs/heads/master'
              with:
                  registry: ${{ env.REGISTRY }}
                  username: ${{ github.actor }}
                  password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}

            - name: Set up QEMU
              uses: docker/setup-qemu-action@v2

            - name: Set up Docker Buildx
              id: buildx
              uses: docker/setup-buildx-action@v2
              with:
                  driver-opts: network=host

            - name: Build and push Docker image (cloud-mt-baseline)
              uses: docker/build-push-action@v3
              if: github.ref == 'refs/heads/master'
              with:
                  context: .
                  file: ./.github/deployment/Dockerfile-mt-cloud
                  push: true
                  tags: ${{ env.LOCAL_CACHE }}
                  target: jwst

            - name: Get current time
              id: time
              run: echo "::set-output name=time::$(date +'%Y%m%d-%H%M')"

            - name: Extract metadata (tags, labels) for Docker (mt-cloud)
              id: meta_mt_cloud
              uses: docker/metadata-action@v4
              with:
                  images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.CLOUD_IMAGE_NAME }}
                  tags: |
                      ${{ env.IMAGE_TAG_MT }}-${{ steps.time.outputs.time }}
                      ${{ env.IMAGE_TAG_MT_LATEST }}

            - name: Build and push Docker image (mt-cloud)
              uses: docker/build-push-action@v3
              with:
                  context: .
                  platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
                  file: ./.github/deployment/Dockerfile-mt-cloud-prod
                  push: ${{ github.ref == 'refs/heads/master' && true || false }}
                  tags: ${{ steps.meta_mt_cloud.outputs.tags }}
                  labels: ${{ steps.meta_mt_cloud.outputs.labels }}
                  target: affine
                  build-args: |
                      BASE_IMAGE=${{ env.LOCAL_CACHE }}

    cloud:
        runs-on: [self-hosted, linux, x64]
        environment: development
        permissions:
            contents: read
            packages: write

        steps:
            - name: Checkout
              uses: actions/checkout@v2

            - name: Install Rust
              uses: actions-rs/toolchain@v1
              with:
                  profile: minimal
                  toolchain: stable
                  override: true

            - uses: Swatinem/rust-cache@v2
            - name: Set up pnpm cache
              uses: actions/cache@v3
              continue-on-error: false
              with:
                  path: ./node_modules/.pnpm-store
                  key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
                  restore-keys: ${{ runner.os }}-pnpm-

            - name: Cargo Vendor
              run: cargo vendor > .cargo/config

            - name: Log in to the Container registry
              uses: docker/login-action@v2
              if: github.ref == 'refs/heads/master'
              with:
                  registry: ${{ env.REGISTRY }}
                  username: ${{ github.actor }}
                  password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}

            - name: Get current time
              id: time
              run: echo "::set-output name=time::$(date +'%H%M')"

            - name: Extract metadata (tags, labels) for Docker (cloud)
              id: meta_cloud
              uses: docker/metadata-action@v4
              with:
                  images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.CLOUD_IMAGE_NAME }}
                  tags: |
                      ${{ env.IMAGE_TAG }}-${{ steps.time.outputs.time }}
                      ${{ env.IMAGE_TAG_LATEST }}

            - name: Set up QEMU
              uses: docker/setup-qemu-action@v2

            - name: Set up Docker Buildx
              id: buildx
              uses: docker/setup-buildx-action@v2
              with:
                  driver-opts: network=host

            - name: Build Docker image (pull request)
              uses: docker/build-push-action@v3
              if: github.ref != 'refs/heads/master'
              with:
                  context: .
                  file: ./.github/deployment/Dockerfile-cloud
                  tags: ${{ steps.meta_cloud.outputs.tags }}
                  labels: ${{ steps.meta_cloud.outputs.labels }}
                  target: cloud

            - name: Build and push Docker image (cloud)
              uses: docker/build-push-action@v3
              if: github.ref == 'refs/heads/master'
              with:
                  context: .
                  file: ./.github/deployment/Dockerfile-cloud
                  push: true
                  tags: ${{ steps.meta_cloud.outputs.tags }}
                  labels: ${{ steps.meta_cloud.outputs.labels }}
                  target: cloud

    jwst:
        runs-on: ubuntu-latest
        environment: development
        permissions:
            contents: read
            packages: write

        steps:
            - name: Checkout
              uses: actions/checkout@v2
              with:
                  fetch-depth: 0

            - uses: pnpm/action-setup@v2
            - name: Install Rust
              uses: actions-rs/toolchain@v1
              with:
                  profile: minimal
                  toolchain: stable
                  target: x86_64-unknown-linux-musl
                  override: true

            - uses: Swatinem/rust-cache@v2
            - name: Set up pnpm cache
              uses: actions/cache@v3
              continue-on-error: false
              with:
                  path: ./node_modules/.pnpm-store
                  key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
                  restore-keys: ${{ runner.os }}-pnpm-

            - name: Build & Check
              run: |
                  sudo apt install -yq musl-tools musl-dev
                  pnpm i --frozen-lockfile --store=node_modules/.pnpm-store
                  cargo vendor > .cargo/config
                  cargo build --profile fast-release --package keck --target x86_64-unknown-linux-musl
              env:
                  CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: x86_64-linux-musl-gcc
                  CARGO_TERM_COLOR: always

            - name: Log in to the Container registry
              if: github.ref == 'refs/heads/master'
              uses: docker/login-action@v2
              with:
                  registry: ${{ env.REGISTRY }}
                  username: ${{ github.actor }}
                  password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}

            - name: Get current time
              id: time
              run: echo "::set-output name=time::$(date +'%H%M')"

            - name: Extract metadata (tags, labels) for Docker
              id: meta_jwst
              uses: docker/metadata-action@v4
              with:
                  images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.JWST_IMAGE_NAME }}
                  tags: |
                      ${{ env.IMAGE_TAG }}-${{ steps.time.outputs.time }}
                      ${{ env.IMAGE_TAG_LATEST }}

            - name: Build and push Docker image (jwst)
              uses: docker/build-push-action@v3
              with:
                  context: .
                  file: ./.github/deployment/Dockerfile-jwst
                  push: ${{ github.ref == 'refs/heads/master' && true || false }}
                  tags: ${{ steps.meta_jwst.outputs.tags }}
                  labels: ${{ steps.meta_jwst.outputs.labels }}
                  target: jwst
                  network: host

    apiproxy:
        if: github.ref == 'refs/heads/master'
        runs-on: ubuntu-latest
        environment: development
        permissions:
            contents: read
            packages: write

        steps:
            - name: Checkout
              uses: actions/checkout@v2

            - name: Log in to the Container registry
              uses: docker/login-action@v2
              with:
                  registry: ${{ env.REGISTRY }}
                  username: ${{ github.actor }}
                  password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}

            - name: Extract metadata (tags, labels) for Docker (apiproxy)
              id: meta_apiproxy
              uses: docker/metadata-action@v4
              with:
                  images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.APIPROXY_IMAGE_NAME }}
                  tags: |
                      ${{ env.IMAGE_TAG }}
                      ${{ env.IMAGE_TAG_LATEST }}

            - name: Build and push Docker image (apiproxy)
              uses: docker/build-push-action@v3
              with:
                  context: .
                  file: ./.github/deployment/Dockerfile-apiproxy
                  push: ${{ github.ref == 'refs/heads/master' && true || false }}
                  tags: ${{ steps.meta_apiproxy.outputs.tags }}
                  labels: ${{ steps.meta_apiproxy.outputs.labels }}
                  target: apiproxy

    lint:
        runs-on: ubuntu-latest
        environment: development
        permissions:
            contents: read
            security-events: write

        steps:
            - name: Checkout
              uses: actions/checkout@v2
              with:
                  fetch-depth: 0

            - name: Install Rust
              uses: actions-rs/toolchain@v1
              with:
                  profile: minimal
                  toolchain: stable
                  components: clippy
                  override: true

            - uses: Swatinem/rust-cache@v2
            - name: Set up pnpm cache
              uses: actions/cache@v3
              continue-on-error: false
              with:
                  path: ./node_modules/.pnpm-store
                  key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
                  restore-keys: ${{ runner.os }}-pnpm-

            - name: Install required cargo components
              uses: taiki-e/install-action@v2
              with:
                  tool: clippy-sarif,sarif-fmt

            - name: Build & Check
              run: |
                  cargo vendor > .cargo/config
                  cargo clippy --all-features --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt
                  RUSTDOCFLAGS="-D rustdoc::broken-intra-doc-links" cargo doc --workspace --all-features --no-deps
              env:
                  CARGO_TERM_COLOR: always

            - name: Upload analysis results to GitHub
              uses: github/codeql-action/upload-sarif@v2
              with:
                  sarif_file: rust-clippy-results.sarif
                  wait-for-processing: true

    test:
        name: test & collect coverage
        runs-on: ubuntu-latest
        continue-on-error: true
        env:
            RUSTFLAGS: -D warnings
            CARGO_TERM_COLOR: always
        steps:
            - uses: actions/checkout@v3
            - uses: dtolnay/rust-toolchain@stable
              with:
                  components: llvm-tools-preview
            - uses: Swatinem/rust-cache@v2

            - name: Install latest nextest release
              uses: taiki-e/install-action@nextest
            - name: Install cargo-llvm-cov
              uses: taiki-e/install-action@cargo-llvm-cov

            - name: Collect coverage data
              run: cargo llvm-cov nextest --lcov --output-path lcov.info
            - name: Upload coverage data to codecov
              uses: codecov/codecov-action@v3
              with:
                  name: tests
                  files: lcov.info

    memory_check:
        name: memory check
        runs-on: ubuntu-latest
        continue-on-error: true
        env:
            RUSTFLAGS: -D warnings -Zsanitizer=address
            ASAN_OPTIONS: detect_leaks=1
            CARGO_TERM_COLOR: always
        steps:
            - uses: actions/checkout@v3
            - uses: dtolnay/rust-toolchain@stable
              with:
                  toolchain: nightly-2023-05-23
            - uses: Swatinem/rust-cache@v2

            - name: Memory Check
              run: |
                  rustup component add rust-src
                  # only check jwst-codec now, yrs has many memory leaks
                  cargo test -Zbuild-std --target x86_64-unknown-linux-gnu -p jwst-codec --lib

    loom:
        name: loom thread test
        runs-on: ubuntu-latest
        continue-on-error: true
        env:
            RUSTFLAGS: --cfg loom
            RUST_BACKTRACE: full
            CARGO_TERM_COLOR: always
        steps:
            - uses: actions/checkout@v3
            - uses: dtolnay/rust-toolchain@stable
            - uses: Swatinem/rust-cache@v2

            - name: Install latest nextest release
              uses: taiki-e/install-action@nextest

            - name: Loom Thread Test
              run: |
                  cargo nextest run -p jwst-codec --lib

    miri:
        name: miri code check
        runs-on: ubuntu-latest
        continue-on-error: true
        env:
            RUST_BACKTRACE: full
            CARGO_TERM_COLOR: always
            MIRIFLAGS: -Zmiri-backtrace=full -Zmiri-tree-borrows
        steps:
            - uses: actions/checkout@v3
            - uses: dtolnay/rust-toolchain@stable
              with:
                  toolchain: nightly-2023-05-23
                  components: miri
            - uses: Swatinem/rust-cache@v2

            - name: Install latest nextest release
              uses: taiki-e/install-action@nextest

            - name: Miri Code Check
              run: |
                  cargo miri nextest run -p jwst-codec -j2

    fuzzing:
        name: fuzzing
        runs-on: ubuntu-latest
        continue-on-error: true
        env:
            RUSTFLAGS: -D warnings
            CARGO_TERM_COLOR: always
        steps:
            - uses: actions/checkout@v3
            - uses: dtolnay/rust-toolchain@master
              with:
                  toolchain: nightly-2023-05-23
            - uses: Swatinem/rust-cache@v2

            - name: fuzzing
              working-directory: ./libs/jwst-codec
              run: |
                  cargo install cargo-fuzz
                  cargo fuzz run apply_update -- -max_total_time=30
                  cargo fuzz run codec_doc_any_struct -- -max_total_time=30
                  cargo fuzz run codec_doc_any -- -max_total_time=30
                  cargo fuzz run decode_bytes -- -max_total_time=30
                  cargo fuzz run i64_decode -- -max_total_time=30
                  cargo fuzz run i64_encode -- -max_total_time=30
                  cargo fuzz run ins_del_text -- -max_total_time=30
                  cargo fuzz run sync_message -- -max_total_time=30
                  cargo fuzz run u64_decode -- -max_total_time=30
                  cargo fuzz run u64_encode -- -max_total_time=30
                  cargo fuzz run apply_update -- -max_total_time=30

            - name: upload fuzz artifacts
              if: ${{ failure() }}
              uses: actions/upload-artifact@v3
              with:
                  name: fuzz-artifact
                  path: ./lib/jwst-codec/fuzz/artifacts/**/*