Merge pull request #1648 from Toktar/ga-ci

adds GitHub Actions and build containers

.github/workflows/README.md

@@ -0,0 +1,25 @@
+### Github Actions Workflow
+
+This build file replaces the existing `Jenkins.ci` build process.
+
+`lint.yaml` replaces the `Static code validation` stage of the Jenkins build.
+
+`build.yaml` replaces the `Build / Test` stage of the Jenkins build.
+
+Many of the other stages are replaced merely by the fact we're using Github Actions, we use prebuild Docker containers so we don't have to replicate the steps for building containers.
+
+The `Build result notification` stage was not moved to GHA, build failures will be reports via GHA.
+
+The build process for `Jenkins.nightly` was not ported to GHA.
+
+#### Configuring actions
+
+If you are cloning or forking this repo you will need to configure two secrets for Actions to run correctly.
+
+Secrets can be set via Settings -> Secrets -> New repository secret.
+
+CR_USER is your GH username.
+CR_PAT can be created by following [these directions](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
+
+Once you have run the build once with those secrets, you have to make then package public.
+Access the package at https://ghcr.io/USER/indy-node/indy-node-build or https://ghcr.io/USER/indy-node/indy-node-lint then change the visibility in 'Package Settings' to 'Public' then re-run the build.

.github/workflows/build.yaml

@@ -0,0 +1,117 @@
+name: indy-node-build
+on: [ push, pull_request ]
+
+jobs:
+  workflow-setup:
+    runs-on: ubuntu-latest
+    outputs:
+      CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }}
+      CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }}
+      # Expose the lowercase version of the GitHub repository name
+      # to all subsequent jobs that reference image repositories
+      # as the push and pull operations require the URL of the repository 
+      # to be in lowercase.
+      GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Set outputs
+        id: cache
+        run: |
+          echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}"
+          echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile') }}"
+          echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})"
+           
+  build-lint-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }}
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-lint
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_LINT }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-lint.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/lint/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} .
+          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest
+          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }}
+
+  build-test-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }}
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-build
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_BUILD }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-build.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/build/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} .
+          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest
+          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }}
+
+  indy_node:
+    name: Build Indy Node
+    needs: build-test-image
+    runs-on: ubuntu-18.04
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-build
+    strategy:
+      matrix:
+        module: [indy_node, indy_common]
+        slice: [1, 2, 3, 4 ,5, 6, 7,8, 9, 10, 11]
+      fail-fast: false
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: pip install .[tests]
+        continue-on-error: true
+
+      - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/${{ strategy.job-total }}
+        run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv --junitxml=test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml" --dir "${{ matrix.module }}" --output "test-result-indy-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/${{ strategy.job-total }}"
+
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        with:
+          check_name: Indy Node ${{ matrix.module }} ${{ matrix.slice }}/${{ strategy.job-total }} Test Report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/indy-node-lint
+    needs: [workflow-setup, build-lint-image]
+    steps:      
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: flake8
+        run: python3 -m flake8

.github/workflows/build/Dockerfile

@@ -0,0 +1,17 @@
+FROM hyperledger/indy-core-baseci:0.0.3-master
+LABEL maintainer="Hyperledger <hyperledger-indy@lists.hyperledger.org>"
+
+RUN apt-get update -y && apt-get install -y \
+    python3-nacl \
+    libindy-crypto=0.4.5 \
+    libindy=1.13.0~1420 \
+# rocksdb python wrapper
+    libbz2-dev \
+    zlib1g-dev \
+    liblz4-dev \
+    libsnappy-dev \
+    rocksdb=5.8.8 \
+    ursa=0.3.2-2 \
+    jq
+
+RUN indy_image_clean

.github/workflows/build/README.md

@@ -0,0 +1,3 @@
+# Building the build image
+
+This `Dockerfile` is used as part of the workflow, any changes to it will force the docker image to be rebuilt and that new image will be used to run the downstream workflow.

.github/workflows/lint/Dockerfile

@@ -0,0 +1,21 @@
+# Development
+FROM ubuntu:18.04
+LABEL maintainer="Kevin Griffin <griffin.kev@gmail.com>"
+
+RUN apt-get update && apt-get dist-upgrade -y
+
+# Install environment
+RUN apt-get install -y \
+	git \
+	wget \
+	python3.5 \
+	python3-pip \
+	python-setuptools \
+	python3-nacl
+
+RUN pip3 install -U \
+	'pip<10.0.0' \
+	setuptools \
+	pep8==1.7.1 \
+	pep8-naming==0.6.1 \
+	flake8==3.5.0

.github/workflows/lint/README.md

@@ -0,0 +1,3 @@
+# Building the lint image
+
+This `Dockerfile` is used as part of the workflow, any changes to it will force the docker image to be rebuilt and that new image will be used to run the downstream workflow.

.gitignore

@@ -24,6 +24,9 @@ var/
 *.egg
 *.eggs
 
+# Needed for GitHub Actions
+!.github/workflows/build
+
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
@@ -72,8 +75,15 @@ include/
 # generated doc files
 docs/source/api_docs/
 
-# hidden files
-.*
+# IntelliJ specific config
+*.idea
+*.iml
+
+#vscode
+.vscode
 
 # Vagrant files
-.vagrant
+.vagrant
+
+# test output from working with GitHub actions
+test-result-node.xml

setup.py

@@ -28,7 +28,7 @@
 BASE_DIR = os.path.join(os.path.expanduser("~"), ".indy")
 
 tests_require = ['attrs==19.1.0', 'pytest==3.3.1', 'pytest-xdist==1.22.1', 'pytest-forked==0.2',
-                 'python3-indy==1.13.0-dev-1420', 'pytest-asyncio==0.8.0']
+                 'python3-indy==1.15.0', 'pytest-asyncio==0.8.0']
 
 setup(
     name=metadata['__title__'],