ci: add cron job for GM

Signed-off-by: spacewander <spacewanderlzx@gmail.com>
apache · Nov 25, 2022 · e35bc87 · e35bc87
1 parent 82c2349
commit e35bc87
Show file tree

Hide file tree

Showing 4 changed files with 231 additions and 30 deletions.
diff --git a/.github/workflows/gm-cron.yaml b/.github/workflows/gm-cron.yaml
@@ -0,0 +1,161 @@
+name: CI GM (cron)
+
+on:
+  schedule:
+    # UTC 7:30 every Friday
+    - cron: "30 7 * * 5"
+# FIXME: remove this once the PR is going to be merged
+  push:
+    branches: [master, 'release/**']
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+  pull_request:
+    branches: [master, 'release/**']
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-20.04
+        os_name:
+          - linux_openresty_tongsuo
+        test_dir:
+          - t/plugin/[a-k]*
+          - t/plugin/[l-z]*
+          - t/[a-m]*
+          - t/node t/pubsub t/[r-x]*
+
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: 90
+    env:
+      SERVER_NAME: ${{ matrix.os_name }}
+      OPENRESTY_VERSION: default
+
+    # TODO: refactor the workflows to reduce duplicate parts. Maybe we can write them in shell
+    # scripts or a separate action?
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3.1.0
+        with:
+          submodules: recursive
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.17"
+
+      - name: Cache deps
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-deps
+        with:
+          path: deps
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.os_name }}-${{ hashFiles('rockspec/apisix-master-0.rockspec') }}
+
+      - name: Cache Tongsuo compilation
+        id: cache-tongsuo
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-tongsuo
+        with:
+          path: ./tongsuo
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.os_name }}-tongsuo-ver
+
+      - name: Test SSL Env
+        id: test_ssl_env
+        shell: bash
+        if: steps.cache-tongsuo.outputs.cache-hit != 'true'
+        run: |
+          echo "compile_tongsuo=true" >>$GITHUB_OUTPUT
+
+      - name: Extract test type
+        shell: bash
+        id: test_env
+        run: |
+          test_dir="${{ matrix.test_dir }}"
+          if [[ $test_dir =~ 't/plugin' ]]; then
+            echo "type=plugin" >>$GITHUB_OUTPUT
+          fi
+          if [[ $test_dir =~ 't/admin ' ]]; then
+            echo "type=first" >>$GITHUB_OUTPUT
+          fi
+          if [[ $test_dir =~ ' t/xrpc' ]]; then
+            echo "type=last" >>$GITHUB_OUTPUT
+          fi
+
+      - name: Linux launch common services
+        run: |
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml
+
+      - name: Start CI env (FIRST_TEST)
+        if: steps.test_env.outputs.type == 'first'
+        run: |
+          # launch deps env
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+
+      - name: Start CI env (PLUGIN_TEST)
+        if: steps.test_env.outputs.type == 'plugin'
+        run: |
+          # download keycloak cas provider
+          sudo wget https://github.com/jacekkow/keycloak-protocol-cas/releases/download/18.0.2/keycloak-protocol-cas-18.0.2.jar -O /opt/keycloak-protocol-cas-18.0.2.jar
+
+          ./ci/pod/openfunction/build-function-image.sh
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+          sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh
+
+      - name: Start CI env (LAST_TEST)
+        if: steps.test_env.outputs.type == 'last'
+        run: |
+          # generating SSL certificates for Kafka
+          sudo keytool -genkeypair -keyalg RSA -dname "CN=127.0.0.1" -alias 127.0.0.1 -keystore ./ci/pod/kafka/kafka-server/selfsigned.jks -validity 365 -keysize 2048 -storepass changeit
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+          sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh
+
+      - name: Start Dubbo Backend
+        if: steps.test_env.outputs.type == 'plugin'
+        run: |
+          sudo apt install -y maven
+          cd t/lib/dubbo-backend
+          mvn package
+          cd dubbo-backend-provider/target
+          java -Djava.net.preferIPv4Stack=true -jar dubbo-demo-provider.one-jar.jar > /tmp/java.log &
+
+      - name: Build xDS library
+        if: steps.test_env.outputs.type == 'last'
+        run: |
+          cd t/xds-library
+          go build -o libxds.so -buildmode=c-shared main.go export.go
+
+      - name: Build wasm code
+        if: steps.test_env.outputs.type == 'last'
+        run: |
+          export TINYGO_VER=0.20.0
+          wget https://github.com/tinygo-org/tinygo/releases/download/v${TINYGO_VER}/tinygo_${TINYGO_VER}_amd64.deb 2>/dev/null
+          sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb
+          cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p
+
+      - name: Linux Before install
+        env:
+          COMPILE_TONGSUO: ${{ steps.test_ssl_env.outputs.compile_tongsuo }}
+        run: |
+          sudo --preserve-env=COMPILE_TONGSUO \
+          ./ci/${{ matrix.os_name }}_runner.sh before_install
+
+      - name: Linux Install
+        run: |
+          sudo --preserve-env=OPENRESTY_VERSION \
+          ./ci/${{ matrix.os_name }}_runner.sh do_install
+
+      - name: Linux Script
+        env:
+          TEST_FILE_SUB_DIR: ${{ matrix.test_dir }}
+        run: sudo -E ./ci/${{ matrix.os_name }}_runner.sh script
diff --git a/.github/workflows/gm.yml b/.github/workflows/gm.yml
@@ -27,9 +27,8 @@ jobs:
         platform:
           - ubuntu-20.04
         os_name:
-          - linux_openresty
+          - linux_openresty_tongsuo
         test_dir:
-          # TODO: cover all tests by adding a CI cron job
           - t/gm
 
     runs-on: ${{ matrix.platform }}
@@ -63,48 +62,27 @@ jobs:
           # See https://github.com/Tongsuo-Project/Tongsuo/issues/318
           key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.os_name }}-tongsuo-ver
 
-      - name: Compile SSL lib
+      - name: Test SSL Env
+        id: test_ssl_env
         shell: bash
         if: steps.cache-tongsuo.outputs.cache-hit != 'true'
         run: |
-          git clone https://github.com/api7/tongsuo --depth 1
-          pushd tongsuo
-          # build binary
-          ./config enable-ntls -static
-          make -j2
-          mv apps/openssl apps/static-openssl
-          ./config shared enable-ntls -g --prefix=/usr/local/tongsuo
-          make -j2
-          popd
-
-      - name: Install SSL lib
-        run: |
-          pushd tongsuo
-          sudo make install_sw
-          sudo cp apps/static-openssl /usr/local/tongsuo/bin/openssl
-          export PATH=/usr/local/tongsuo/bin:$PATH
-          openssl version
-          popd
+          echo "compile_tongsuo=true" >>$GITHUB_OUTPUT
 
       - name: Linux launch common services
         run: |
           make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml
 
       - name: Linux Before install
+        env:
+          COMPILE_TONGSUO: ${{ steps.test_ssl_env.outputs.compile_tongsuo }}
         run: |
-          sudo ./ci/${{ matrix.os_name }}_runner.sh before_install
+          sudo --preserve-env=COMPILE_TONGSUO \
+          ./ci/${{ matrix.os_name }}_runner.sh before_install
 
       - name: Linux Do install
         run: |
-          export OR_PREFIX=/usr/local/openresty-debug
-          export openssl_prefix=/usr/local/tongsuo
-          export zlib_prefix=$OR_PREFIX/zlib
-          export pcre_prefix=$OR_PREFIX/pcre
-
-          export cc_opt="-DNGX_LUA_ABORT_AT_PANIC -I${zlib_prefix}/include -I${pcre_prefix}/include -I${openssl_prefix}/include"
-          export ld_opt="-L${zlib_prefix}/lib -L${pcre_prefix}/lib -L${openssl_prefix}/lib64 -Wl,-rpath,${zlib_prefix}/lib:${pcre_prefix}/lib:${openssl_prefix}/lib64"
           sudo --preserve-env=OPENRESTY_VERSION \
-            --preserve-env=cc_opt,ld_opt \
             ./ci/${{ matrix.os_name }}_runner.sh do_install
 
       - name: Linux Script

diff --git a/ci/linux_openresty_tongsuo_runner.sh b/ci/linux_openresty_tongsuo_runner.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+export OPENRESTY_VERSION=source
+export SSL_LIB_VERSION=tongsuo
+
+
+before_install() {
+    if [ -n "$COMPILE_TONGSUO" ]; then
+        git clone https://github.com/api7/tongsuo --depth 1
+        pushd tongsuo
+        # build binary
+        ./config enable-ntls -static
+        make -j2
+        mv apps/openssl apps/static-openssl
+        ./config shared enable-ntls -g --prefix=/usr/local/tongsuo
+        make -j2
+        popd
+    fi
+
+    pushd tongsuo
+    sudo make install_sw
+    sudo cp apps/static-openssl /usr/local/tongsuo/bin/openssl
+    export PATH=/usr/local/tongsuo/bin:$PATH
+    openssl version
+    popd
+}
+
+
+case_opt=$1
+
+case ${case_opt} in
+before_install)
+    before_install
+    ;;
+esac
+
+. ./ci/linux_openresty_common_runner.sh
diff --git a/utils/linux-install-openresty.sh b/utils/linux-install-openresty.sh
@@ -35,6 +35,7 @@ abt_branch=${abt_branch:="master"}
 COMPILE_OPENSSL3=${COMPILE_OPENSSL3-no}
 USE_OPENSSL3=${USE_OPENSSL3-no}
 OPENSSL3_PREFIX=${OPENSSL3_PREFIX-/home/runner}
+SSL_LIB_VERSION=${SSL_LIB_VERSION-openssl}
 
 if [ "$OPENRESTY_VERSION" == "source" ]; then
     if [ "$COMPILE_OPENSSL3" == "yes" ]; then
@@ -57,6 +58,15 @@ if [ "$OPENRESTY_VERSION" == "source" ]; then
         export ld_opt="-L$OPENSSL3_PREFIX/openssl-3.0/lib64 -Wl,-rpath,$OPENSSL3_PREFIX/openssl-3.0/lib64"
     fi
 
+    if [ "$SSL_LIB_VERSION" == "tongsuo" ]; then
+        export openssl_prefix=/usr/local/tongsuo
+        export zlib_prefix=$OPENRESTY_PREFIX/zlib
+        export pcre_prefix=$OPENRESTY_PREFIX/pcre
+
+        export cc_opt="-DNGX_LUA_ABORT_AT_PANIC -I${zlib_prefix}/include -I${pcre_prefix}/include -I${openssl_prefix}/include"
+        export ld_opt="-L${zlib_prefix}/lib -L${pcre_prefix}/lib -L${openssl_prefix}/lib64 -Wl,-rpath,${zlib_prefix}/lib:${pcre_prefix}/lib:${openssl_prefix}/lib64"
+    fi
+
     cd ..
     wget https://raw.githubusercontent.com/api7/apisix-build-tools/$abt_branch/build-apisix-base.sh
     chmod +x build-apisix-base.sh