validate-release.yml

name: "Validate Apache Release"

on:
  workflow_dispatch:
    inputs:
      release_version:
        required: true
        description: check version
        default: '1.3.0'
      gpg_user:
        required: true
        description: release gpg user
        default: 'imbajin'

  push:
    branches:
      - 'release-*'
  pull_request:
    branches:
      - 'release-*'

jobs:
  build:
    name: "Build On ${{ matrix.os }} (java-${{ matrix.java_version }})"
    runs-on: ${{ matrix.os }}
    env:
      SCRIPT_PATH: hugegraph-dist/scripts/
      URL_PREFIX: https://dist.apache.org/repos/dist/dev/incubator/hugegraph/
      USER: ${{ inputs.gpg_user }}
      # TODO: parse version from the running branch name & also adapt the input version
      RELEASE_VERSION: ''
      USE_STAGE: 'true' # Whether to include the stage repository.
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Install JDK ${{ matrix.java_version }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java_version }}
          distribution: 'adopt'
      - name: Cache Maven packages
        uses: actions/cache@v3
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - name: Get Yarn path
        id: yarn-cache-dir-path
        run: echo "::set-output name=dir::$(yarn cache dir)"
      - name: Cache Yarn packages
        uses: actions/cache@v3
        # use id to check `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
        id: yarn-cache
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Use staged maven repo settings
        if: ${{ env.USE_STAGE == 'true' }}
        run: |
          cp $HOME/.m2/settings.xml /tmp/settings.xml
          cp -vf .github/workflows/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml

      - name: 1. Download SVN Sources
        run: |
          rm -rf dist/${{ inputs.release_version }}
          svn co ${URL_PREFIX}/${{ inputs.release_version }} dist/${{ inputs.release_version }}

      - name: 2. Check Environment & Import Public Keys
        run: |
          cd dist/${{ inputs.release_version }} || exit

          shasum --version 1>/dev/null || exit
          gpg --version 1>/dev/null || exit

          wget https://downloads.apache.org/incubator/hugegraph/KEYS || exit
          echo "Import KEYS:" && gpg --import KEYS
          # TODO: how to trust all public keys in gpg list, currently only trust the first one
          echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key $USER trust

          echo "trust all pk"
          for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do
              echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust
          done

      - name: 3. Check SHA512 & GPG Signature
        run: |
          cd dist/${{ inputs.release_version }} || exit

          for i in *.tar.gz; do
            echo "$i"
            shasum -a 512 --check "$i".sha512 || exit
            eval gpg "${GPG_OPT}" --verify "$i".asc "$i" || exit
          done

      - name: 4. Validate Source Packages
        run: |
          cd dist/${{ inputs.release_version }} || exit

          ls -lh ./*.tar.gz
          CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial"
          CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY"
          for i in *src.tar.gz; do
            echo "$i"

            # 4.1 check the directory name include "incubating"
            if [[ ! "$i" =~ "incubating" ]]; then
              echo "The package name $i should include incubating" && exit 1
            fi

            tar xzvf "$i" || exit
            pushd "$(basename "$i" .tar.gz)" || exit
            echo "Start to check the package content: $(basename "$i" .tar.gz)"

            # 4.2 check the directory include "NOTICE" and "LICENSE" and "DISCLAIMER" file
            if [[ ! -f "LICENSE" ]]; then
              echo "The package should include LICENSE file" && exit 1
            fi
            if [[ ! -f "NOTICE" ]]; then
              echo "The package should include NOTICE file" && exit 1
            fi
            if [[ ! -f "DISCLAIMER" ]]; then
                echo "The package should include DISCLAIMER file" && exit 1
            fi

            # 4.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE and NOTICE files
            COUNT=$(grep -E $CATEGORY_X LICENSE NOTICE | wc -l)
            if [[ $COUNT -ne 0 ]]; then
              grep -E "$CATEGORY_X" LICENSE NOTICE
              echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1
            fi
          
            # 4.4: ensure doesn't contains ASF CATEGORY B License dependencies in LICENSE and NOTICE files
            COUNT=$(grep -E $CATEGORY_B LICENSE NOTICE | wc -l)
            if [[ $COUNT -ne 0 ]]; then
              grep -E "$CATEGORY_B" LICENSE NOTICE
              echo "The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT" && exit 1
            fi

            # 4.5 ensure doesn't contains empty directory or file
            find . -type d -empty | while read -r EMPTY_DIR; do
              find . -type d -empty
              echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1
            done
            find . -type f -empty | while read -r EMPTY_FILE; do
              find . -type f -empty
              echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1
            done

            # 4.6 ensure any file should less than 800kb
            find . -type f -size +800k | while read -r FILE; do
              find . -type f -size +800k
              echo "The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb" && exit 1
            done

            # 4.7: ensure all binary files are documented in LICENSE
            find . -type f | perl -lne 'print if -B' | while read -r BINARY_FILE; do
              FILE_NAME=$(basename "$BINARY_FILE")
              if grep -q "$FILE_NAME" LICENSE; then
                echo "Binary file $BINARY_FILE is documented in LICENSE, please check manually"
              else
                echo "Error: Binary file $BINARY_FILE is not documented in LICENSE" && exit 1
              fi
            done

            # 4.8 test compile the packages
            if [[ ${{ matrix.java_version }} == 8 && "$i" =~ "computer" ]]; then
              echo "skip computer module in java8"
              popd || exit
              continue
            fi
            # TODO: consider using commands that are entirely consistent with building binary packages
            mvn package -DskipTests -Papache-release -ntp -e || exit
            ls -lh

            popd || exit
          done

      - name: 5. Run Compiled Packages In Server
        run: |
          cd dist/${{ inputs.release_version }} || exit

          ls -lh
          pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*${{ inputs.release_version }} || exit
          bin/init-store.sh || exit
          sleep 3
          bin/start-hugegraph.sh || exit
          popd || exit

      - name: 6. Run Compiled Packages In ToolChain (Loader & Tool & Hubble)
        run: |
          cd dist/${{ inputs.release_version }} || exit

          pushd ./*toolchain*src || exit
          ls -lh
          pushd ./*toolchain*${{ inputs.release_version }} || exit
          ls -lh

          # 6.1 load some data first
          echo "test loader"
          pushd ./*loader*${{ inputs.release_version }} || exit
          bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \
            -g hugegraph || exit
          popd || exit

          # 6.2 try some gremlin query & api in tool
          echo "test tool"
          pushd ./*tool*${{ inputs.release_version }} || exit
          bin/hugegraph gremlin-execute --script 'g.V().count()' || exit
          bin/hugegraph task-list || exit
          bin/hugegraph backup -t all --directory ./backup-test || exit
          popd || exit

          # 6.3 start hubble and connect to server 
          echo "test hubble"
          pushd ./*hubble*${{ inputs.release_version }} || exit
          # TODO: add hubble doc & test it
          cat conf/hugegraph-hubble.properties
          bin/start-hubble.sh || exit
          bin/stop-hubble.sh || exit
          popd || exit

          popd || exit
          popd || exit
          # stop server
          pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*${{ inputs.release_version }} || exit
          bin/stop-hugegraph.sh || exit
          popd || exit

          # clear source packages
          rm -rf ./*src*
          ls -lh

      - name: 7. Validate Binary Packages
        run: |
          cd dist/${{ inputs.release_version }} || exit
          CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial"
          for i in *.tar.gz; do
            if [[ "$i" == *-src.tar.gz ]]; then
              # skip source packages
              continue
            fi

            echo "$i"

            # 7.1 check the directory name include "incubating"
            if [[ ! "$i" =~ "incubating" ]]; then
              echo "The package name $i should include incubating" && exit 1
            fi

            tar xzvf "$i" || exit
            pushd "$(basename "$i" .tar.gz)" || exit
            ls -lh
            echo "Start to check the package content: $(basename "$i" .tar.gz)"

            # 7.2 check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" & "licenses" dir
            if [[ ! -f "LICENSE" ]]; then
              echo "The package should include LICENSE file" && exit 1
            fi
            if [[ ! -f "NOTICE" ]]; then
              echo "The package should include NOTICE file" && exit 1
            fi
            if [[ ! -f "DISCLAIMER" ]]; then
                echo "The package should include DISCLAIMER file" && exit 1
            fi
            if [[ ! -d "licenses" ]]; then
              echo "The package should include licenses dir" && exit 1
            fi

            # 7.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE/NOTICE and licenses/* files
            COUNT=$(grep -r -E $CATEGORY_X LICENSE NOTICE licenses | wc -l)
            if [[ $COUNT -ne 0 ]]; then
              grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses
              echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1
            fi

            # 7.4: ensure doesn't contains empty directory or file
            find . -type d -empty | while read -r EMPTY_DIR; do
              find . -type d -empty
              echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1
            done
            find . -type f -empty | while read -r EMPTY_FILE; do
              find . -type f -empty
              echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1
            done

            popd || exit
          done

      - name: 8. Run Binary Packages In Server
        run: |
          cd dist/${{ inputs.release_version }} || exit

          pushd ./*hugegraph-incubating*${{ inputs.release_version }} || exit
          bin/init-store.sh || exit
          sleep 3
          bin/start-hugegraph.sh || exit
          popd || exit

      - name: 9. Run Binary Packages In ToolChain (Loader & Tool & Hubble)
        run: |
          cd dist/${{ inputs.release_version }} || exit

          pushd ./*toolchain*${{ inputs.release_version }} || exit
          ls -lh

          # 9.1 loader some data first
          echo "test loader"
          pushd ./*loader*${{ inputs.release_version }} || exit
          bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \
            -g hugegraph || exit
          popd || exit

          # 9.2 try some gremlin query & api in tool
          echo "test tool"
          pushd ./*tool*${{ inputs.release_version }} || exit
          bin/hugegraph gremlin-execute --script 'g.V().count()' || exit
          bin/hugegraph task-list || exit
          bin/hugegraph backup -t all --directory ./backup-test || exit
          popd || exit

          # 9.3 start hubble and connect to server
          echo "test hubble"
          pushd ./*hubble*${{ inputs.release_version }} || exit
          # TODO: add hubble doc & test it
          cat conf/hugegraph-hubble.properties
          bin/start-hubble.sh || exit
          bin/stop-hubble.sh || exit
          popd || exit

          popd || exit
          # stop server
          pushd ./*hugegraph-incubating*${{ inputs.release_version }} || exit
          bin/stop-hugegraph.sh || exit
          popd || exit

    strategy:
      fail-fast: false
      matrix:
        java_version: [ '8','11' ]
        # TODO: support windows-latest or other OS in future
        os: [ubuntu-latest, macos-latest]