diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml index 270ee46dd13c..ab18725be838 100644 --- a/.github/workflows/gaia.yml +++ b/.github/workflows/gaia.yml @@ -101,6 +101,7 @@ jobs: - name: Upload GIE log if: failure() uses: actions/upload-artifact@v3 + continue-on-error: true with: name: gie-log path: /var/log/graphscope diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml index 9abecdaef754..157174f022f8 100644 --- a/.github/workflows/gss.yml +++ b/.github/workflows/gss.yml @@ -184,8 +184,7 @@ jobs: minikube image load ${{ env.GSS_IMAGE }}:${SHORT_SHA} - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 + - uses: dashanji/kubernetes-log-export-action@v4 env: SHOW_TIMESTAMPS: 'true' OUTPUT_DIR: ${{ github.workspace }}/helm-installation-logs @@ -203,19 +202,6 @@ jobs: helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store helm test ci --timeout 5m0s - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop - - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/demo-fresh-of-helm-installation-logs - NAMESPACES: "gs*,default" - MODE: start - - name: Test the helm deployment run: | # 1. get gss service endpoint @@ -248,19 +234,6 @@ jobs: python3 setup.py build_proto python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_fresh - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop - - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/demo-script-of-helm-installation-with-pv-logs - NAMESPACES: "gs*,default" - MODE: start - - name: restart helm and run demo with the PersistentVolume run: | # restart helm and run demo with the PersistentVolume @@ -269,19 +242,6 @@ jobs: cd ${GITHUB_WORKSPACE}/charts helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop - - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/helm-test-of-helm-installation-with-pv-logs - NAMESPACES: "gs*,default" - MODE: start - - name: Helm Test with Helm Deployment and PersistentVolume run: | # helm test and python test on the restarted store @@ -290,19 +250,6 @@ jobs: export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") helm test ci --timeout 10m0s - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop - - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/demo-after-restart-of-helm-installation-with-pv-logs - NAMESPACES: "gs*,default" - MODE: start - - name: Python Test with Helm Deployment and PersistentVolume run: | export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend) @@ -311,14 +258,15 @@ jobs: cd ${GITHUB_WORKSPACE}/python python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_after_restart + - uses: dashanji/kubernetes-log-export-action@v4 + env: + MODE: stop + - name: upload the k8s logs to artifact if: ${{ always() }} uses: actions/upload-artifact@v3 + continue-on-error: true with: name: k8s-test-logs path: ${{ github.workspace }}/k8s-ci-*-logs - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index 68886d46e87b..570cddf4f10c 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -377,23 +377,6 @@ jobs: # Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space to pull the builder make graphscope CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - - name: Prepare the log directory - run: | - # create the helm installation log directory - mkdir -p ${{ github.workspace }}/helm-installation-logs - - # create the store service of helm installation log directory - mkdir -p ${{ github.workspace }}/store-service-of-helm-installation-logs - - # create the demo script of helm installation log directory - mkdir -p ${{ github.workspace }}/demo-script-of-helm-installation-logs - - # create the store service of kubernetes test log directory - mkdir -p ${{ github.workspace }}/store-service-of-kubernetes-test-logs - - # create the demo script of hdfs test log directory - mkdir -p ${{ github.workspace }}/demo-script-of-hdfs-test-logs - - name: Install Python dependencies run: | cd ${GITHUB_WORKSPACE}/python @@ -448,8 +431,7 @@ jobs: minikube image load graphscope/learning:${SHORT_SHA} echo "loaded learning" - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 + - uses: dashanji/kubernetes-log-export-action@v4 env: SHOW_TIMESTAMPS: 'true' OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-helm-installation-logs @@ -467,19 +449,6 @@ jobs: ./graphscope helm test graphscope --timeout 5m0s - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop - - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-of-helm-installation-logs - NAMESPACES: "gs*,default" - MODE: start - - name: Test demo script of helm installation run: | export NODE_IP=$(kubectl get pod -lgraphscope.coordinator.name=coordinator-graphscope -ojsonpath="{.items[0].status.hostIP}") @@ -490,20 +459,17 @@ jobs: python3 -m pytest -s -vvv ./graphscope/tests/kubernetes/test_demo_script.py -k test_helm_installation - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 + - uses: dashanji/kubernetes-log-export-action@v4 env: MODE: stop - name: Delete Helm Cluster - run: | - helm delete graphscope + run: helm delete graphscope - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 + - uses: dashanji/kubernetes-log-export-action@v4 env: SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-store-service-of-kubernetes-test-logs + OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-test-logs NAMESPACES: "gs*,default" MODE: start @@ -522,25 +488,12 @@ jobs: --cov-report=term --exitfirst -s -vvv --log-cli-level=INFO \ ./graphscope/tests/kubernetes - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop - - name: Upload Coverage uses: codecov/codecov-action@v3 with: file: ./python/coverage.xml fail_ci_if_error: false - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-of-hdfs-test-logs - NAMESPACES: "gs*,default" - MODE: start - - name: HDFS test env: JAVA_HOME: /usr/lib/jvm/default-java @@ -561,28 +514,24 @@ jobs: hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_0_0.csv /ldbc_sample/person_0_0.csv hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_knows_person_0_0.csv /ldbc_sample/person_knows_person_0_0.csv - # validate hadoop - hadoop fs -ls /ldbc_sample - # prepare CI environments export HDFS_TEST_DIR=hdfs:///ldbc_sample export HDFS_HOST=$(hostname -I | awk '{print $1}') # run test cd ${GITHUB_WORKSPACE}/python - python3 -m pytest -s -vvv ./graphscope/tests/kubernetes/test_demo_script.py -k test_demo_on_hdfs # Check the result file have successfully written to the given location # hdfs dfs -test -e /ldbc_sample/res.csv_0 && hdfs dfs -test -e /ldbc_sample/res.csv_1 - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 + - uses: dashanji/kubernetes-log-export-action@v4 env: MODE: stop - name: Upload the k8s logs to artifact uses: actions/upload-artifact@v3 if: ${{ always() }} + continue-on-error: true with: name: k8s-test-logs path: ${{ github.workspace }}/k8s-ci-*-logs @@ -718,11 +667,6 @@ jobs: -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive-experimental:${SHORT_SHA} \ -f .github/workflows/docker/interactive-experimental-local-artifacts.Dockerfile . - - name: Prepare the log directory - run: | - # create the k8s failover test log directory - mkdir -p ${{ github.workspace }}/k8s-failover-test-logs - - name: Prepare Cluster and Data env: GS_TEST_DIR: ${{ github.workspace }}/gstest @@ -743,8 +687,7 @@ jobs: # install python gremlin client pip install gremlinpython - - name: Start to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 + - uses: dashanji/kubernetes-log-export-action@v4 env: SHOW_TIMESTAMPS: 'true' OUTPUT_DIR: ${{ github.workspace }}/k8s-failover-logs @@ -769,14 +712,14 @@ jobs: # run failover test cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_k8s_failover_ci.sh default test-gie-standalone 2 1 + - uses: dashanji/kubernetes-log-export-action@v4 + env: + MODE: stop + - name: upload the k8s logs to artifact uses: actions/upload-artifact@v3 + continue-on-error: true if: ${{ always() }} with: name: k8s-failover-logs path: ${{ github.workspace }}/k8s-failover-logs - - - name: Stop to export kubernetes logs - uses: dashanji/kubernetes-log-export-action@v4 - env: - MODE: stop diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml index 2599da220bc3..5423d49efba6 100644 --- a/.github/workflows/local-ci.yml +++ b/.github/workflows/local-ci.yml @@ -313,6 +313,7 @@ jobs: - name: Upload GIE log if: failure() uses: actions/upload-artifact@v3 + continue-on-error: true with: name: gie-log path: ~/.local/log/graphscope