From 6f90b08c503e9b5a80420d9c5beac1a787731711 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 4 Feb 2021 15:39:40 -0500 Subject: [PATCH] adds GitHub Actions and build containers Signed-off-by: Kevin Griffin --- .github/workflows/README.md | 25 ++++++ .github/workflows/build.yaml | 117 +++++++++++++++++++++++++++++ .github/workflows/build/Dockerfile | 17 +++++ .github/workflows/build/README.md | 3 + .github/workflows/lint/Dockerfile | 21 ++++++ .github/workflows/lint/README.md | 3 + .gitignore | 16 +++- setup.py | 2 +- 8 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/build/Dockerfile create mode 100644 .github/workflows/build/README.md create mode 100644 .github/workflows/lint/Dockerfile create mode 100644 .github/workflows/lint/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..f1587ac39 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,25 @@ +### Github Actions Workflow + +This build file replaces the existing `Jenkins.ci` build process. + +`lint.yaml` replaces the `Static code validation` stage of the Jenkins build. + +`build.yaml` replaces the `Build / Test` stage of the Jenkins build. + +Many of the other stages are replaced merely by the fact we're using Github Actions, we use prebuild Docker containers so we don't have to replicate the steps for building containers. + +The `Build result notification` stage was not moved to GHA, build failures will be reports via GHA. + +The build process for `Jenkins.nightly` was not ported to GHA. + +#### Configuring actions + +If you are cloning or forking this repo you will need to configure two secrets for Actions to run correctly. + +Secrets can be set via Settings -> Secrets -> New repository secret. + +CR_USER is your GH username. +CR_PAT can be created by following [these directions](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) + +Once you have run the build once with those secrets, you have to make then package public. +Access the package at https://ghcr.io/USER/indy-node/indy-node-build or https://ghcr.io/USER/indy-node/indy-node-lint then change the visibility in 'Package Settings' to 'Public' then re-run the build. diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..9ee5ffad3 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,117 @@ +name: indy-node-build +on: [ push, pull_request ] + +jobs: + workflow-setup: + runs-on: ubuntu-latest + outputs: + CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }} + CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }} + # Expose the lowercase version of the GitHub repository name + # to all subsequent jobs that reference image repositories + # as the push and pull operations require the URL of the repository + # to be in lowercase. + GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }} + steps: + - name: Git checkout + uses: actions/checkout@v2 + - name: Set outputs + id: cache + run: | + echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}" + echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile') }}" + echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})" + + build-lint-image: + needs: workflow-setup + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: 1 + CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }} + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + steps: + - name: Git checkout + uses: actions/checkout@v2 + - name: Try load from cache. + id: cache-image-lint + uses: actions/cache@v2 + with: + path: ${GITHUB_WORKSPACE}/cache + key: ${{ env.CACHE_KEY_LINT }} + - name: If NOT found in cache, build and push image. + if: steps.cache-image-lint.outputs.cache-hit != 'true' + run: | + echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin + docker build -f .github/workflows/lint/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} . + docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest + docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest + mkdir -p ${GITHUB_WORKSPACE}/cache + touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }} + + build-test-image: + needs: workflow-setup + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: 1 + CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + steps: + - name: Git checkout + uses: actions/checkout@v2 + - name: Try load from cache. + id: cache-image-build + uses: actions/cache@v2 + with: + path: ${GITHUB_WORKSPACE}/cache + key: ${{ env.CACHE_KEY_BUILD }} + - name: If NOT found in cache, build and push image. + if: steps.cache-image-build.outputs.cache-hit != 'true' + run: | + echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin + docker build -f .github/workflows/build/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} . + docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest + docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest + mkdir -p ${GITHUB_WORKSPACE}/cache + touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }} + + indy_node: + name: Build Indy Node + needs: build-test-image + runs-on: ubuntu-18.04 + container: + image: ghcr.io/${{ github.repository }}/indy-node-build + strategy: + matrix: + module: [indy_node, indy_common] + slice: [1, 2, 3, 4 ,5, 6, 7,8, 9, 10, 11] + fail-fast: false + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Install dependencies + run: pip install .[tests] + continue-on-error: true + + - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/${{ strategy.job-total }} + run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv --junitxml=test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml" --dir "${{ matrix.module }}" --output "test-result-indy-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/${{ strategy.job-total }}" + + - name: Publish Test Report + uses: scacap/action-surefire-report@v1 + with: + check_name: Indy Node ${{ matrix.module }} ${{ matrix.slice }}/${{ strategy.job-total }} Test Report + github_token: ${{ secrets.GITHUB_TOKEN }} + report_paths: test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml + + lint: + name: Lint + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/indy-node-lint + needs: [workflow-setup, build-lint-image] + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: flake8 + run: python3 -m flake8 diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile new file mode 100644 index 000000000..0a1ce13c6 --- /dev/null +++ b/.github/workflows/build/Dockerfile @@ -0,0 +1,17 @@ +FROM hyperledger/indy-core-baseci:0.0.3-master +LABEL maintainer="Hyperledger " + +RUN apt-get update -y && apt-get install -y \ + python3-nacl \ + libindy-crypto=0.4.5 \ + libindy=1.13.0~1420 \ +# rocksdb python wrapper + libbz2-dev \ + zlib1g-dev \ + liblz4-dev \ + libsnappy-dev \ + rocksdb=5.8.8 \ + ursa=0.3.2-2 \ + jq + +RUN indy_image_clean diff --git a/.github/workflows/build/README.md b/.github/workflows/build/README.md new file mode 100644 index 000000000..984f201f2 --- /dev/null +++ b/.github/workflows/build/README.md @@ -0,0 +1,3 @@ +# Building the build image + +This `Dockerfile` is used as part of the workflow, any changes to it will force the docker image to be rebuilt and that new image will be used to run the downstream workflow. \ No newline at end of file diff --git a/.github/workflows/lint/Dockerfile b/.github/workflows/lint/Dockerfile new file mode 100644 index 000000000..d1870d403 --- /dev/null +++ b/.github/workflows/lint/Dockerfile @@ -0,0 +1,21 @@ +# Development +FROM ubuntu:18.04 +LABEL maintainer="Kevin Griffin " + +RUN apt-get update && apt-get dist-upgrade -y + +# Install environment +RUN apt-get install -y \ + git \ + wget \ + python3.5 \ + python3-pip \ + python-setuptools \ + python3-nacl + +RUN pip3 install -U \ + 'pip<10.0.0' \ + setuptools \ + pep8==1.7.1 \ + pep8-naming==0.6.1 \ + flake8==3.5.0 diff --git a/.github/workflows/lint/README.md b/.github/workflows/lint/README.md new file mode 100644 index 000000000..6b2a83fd0 --- /dev/null +++ b/.github/workflows/lint/README.md @@ -0,0 +1,3 @@ +# Building the lint image + +This `Dockerfile` is used as part of the workflow, any changes to it will force the docker image to be rebuilt and that new image will be used to run the downstream workflow. \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6147ec972..ca783e72d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ var/ *.egg *.eggs +# Needed for GitHub Actions +!.github/workflows/build + # Installer logs pip-log.txt pip-delete-this-directory.txt @@ -72,8 +75,15 @@ include/ # generated doc files docs/source/api_docs/ -# hidden files -.* +# IntelliJ specific config +*.idea +*.iml + +#vscode +.vscode # Vagrant files -.vagrant \ No newline at end of file +.vagrant + +# test output from working with GitHub actions +test-result-node.xml \ No newline at end of file diff --git a/setup.py b/setup.py index 5e8cd71e7..e8fdb3788 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ BASE_DIR = os.path.join(os.path.expanduser("~"), ".indy") tests_require = ['attrs==19.1.0', 'pytest==3.3.1', 'pytest-xdist==1.22.1', 'pytest-forked==0.2', - 'python3-indy==1.13.0-dev-1420', 'pytest-asyncio==0.8.0'] + 'python3-indy==1.15.0', 'pytest-asyncio==0.8.0'] setup( name=metadata['__title__'],