Adds config for gihub actions, introducing three jobs. Jobs for indy-…

…common/node and lint.

Signed-off-by: Kevin Griffin <griffin.kev@gmail.com>

.github/workflows/README.md

@@ -0,0 +1,13 @@
+### Github Actions Workflow
+
+This build file replaces the existing `Jenkins.ci` build process.
+
+`lint.yaml` replaces the `Static code validation` stage of the Jenkins build.
+
+`build.yaml` replaces the `Build / Test` stage of the Jenkins build.
+
+Many of the other stages are replaced merely by the fact we're using Github Actions, we use prebuild Docker containers so we don't have to replicate the steps for building containers.
+
+The `Build result notification` stage was not moved to GHA, build failures will be reports via GHA.
+
+The build process for `Jenkins.nightly` was not ported to GHA.

.github/workflows/build.yaml

@@ -0,0 +1,158 @@
+name: indy-node-build
+on: [ push, pull_request ]
+
+jobs:
+  workflow-setup:
+    runs-on: ubuntu-latest
+    outputs:
+      CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }}
+      CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Set outputs
+        id: cache
+        run: |
+          echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}"
+          echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile') }}"
+           
+  build-lint-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-lint
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_LINT }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-lint.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/lint/Dockerfile --no-cache -t ${GITHUB_REPOSITORY}/indy-node-lint:${{ env.CACHE_KEY_LINT }} .
+          docker tag ${GITHUB_REPOSITORY}/indy-node-lint:${{ env.CACHE_KEY_LINT }} ghcr.io/${GITHUB_REPOSITORY}/indy-node-lint:latest
+          docker push ghcr.io/${GITHUB_REPOSITORY}/indy-node-lint:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }}
+
+  build-test-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-build
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_BUILD }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-build.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/build/Dockerfile --no-cache -t ${GITHUB_REPOSITORY}/indy-node-build:${{ env.CACHE_KEY_BUILD }} .
+          docker tag ${GITHUB_REPOSITORY}/indy-node-build:${{ env.CACHE_KEY_BUILD }} ghcr.io/${GITHUB_REPOSITORY}/indy-node-build:latest
+          docker push ghcr.io/${GITHUB_REPOSITORY}/indy-node-build:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }}
+
+
+  decorator-checks:
+    name: Run decorator checks
+    needs: 
+      - build-test-image
+      - lint
+    runs-on: ubuntu-18.04
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-build
+    outputs:
+      matrix-common: ${{ steps.ctd.outputs.matrix-common }}
+      matrix-node: ${{ steps.ctd.outputs.matrix-node }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Check test decorators
+        id: ctd
+        run: |
+          chmod +x ./scripts/run_pytest_check.sh
+          ./scripts/run_pytest_check.sh
+
+  indy_common:
+    name: Build Indy Common
+    needs: decorator-checks
+    runs-on: ubuntu-18.04
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-build
+    strategy:
+      matrix: ${{fromJson(needs.decorator-checks.outputs.matrix-common)}}
+      fail-fast: false
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: pip install .[tests]
+        continue-on-error: true
+
+      - name: Run Indy Common tests
+        run: python3 -m pytest -l -m ${{ matrix.module }} -vv --junitxml=test-result-common-${{ matrix.module }}.xml indy_common
+
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        with:
+          check_name: Indy Common ${{ matrix.module }} Test Report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: test-result-common-${{ matrix.module }}.xml
+
+  indy_node:
+    name: Build Indy Node
+    needs: decorator-checks
+    runs-on: ubuntu-18.04
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-build
+    strategy:
+      matrix: ${{fromJson(needs.decorator-checks.outputs.matrix-node)}}
+      fail-fast: false
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: pip install .[tests]
+        continue-on-error: true
+
+      - name: Run Indy Node ${{ matrix.module }} tests
+        run: python3 -m pytest -l -m ${{ matrix.module }} -vv --junitxml=test-result-node-${{ matrix.module }}.xml indy_node
+
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        with:
+          check_name: Indy Node ${{ matrix.module }} Test Report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: test-result-node-${{ matrix.module }}.xml
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-lint
+    needs:
+      - build-lint-image
+    steps:      
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: flake8
+        run: python3 -m flake8
+

.github/workflows/build/Dockerfile

@@ -0,0 +1,17 @@
+FROM hyperledger/indy-core-baseci:0.0.3-master
+LABEL maintainer="Hyperledger <hyperledger-indy@lists.hyperledger.org>"
+
+RUN apt-get update -y && apt-get install -y \
+    python3-nacl \
+    libindy-crypto=0.4.5 \
+    libindy=1.13.0~1420 \
+# rocksdb python wrapper
+    libbz2-dev \
+    zlib1g-dev \
+    liblz4-dev \
+    libsnappy-dev \
+    rocksdb=5.8.8 \
+    ursa=0.3.2-2 \
+    jq
+
+RUN indy_image_clean

.github/workflows/build/README.md

@@ -0,0 +1,17 @@
+## Indy Node build container
+
+This Docker container replaces the `ubuntu.dockerfile` and is used by the GHA workflow for building and testing Indy Node.
+
+
+## Managing this container
+
+```
+docker build .
+```
+
+```
+docker tag VERSION NAMESPACE/indy-node-build:TAG_NAME
+```
+
+```
+docker push NAMESPACE/indy-node-build:TAG_NAME

.github/workflows/build/indy-core-repo.preferences

@@ -0,0 +1,3 @@
+Package: /indy-crypto/ /libindy/
+Pin: release l=Indy Main Repository
+Pin-Priority: 1000

.github/workflows/build/scripts/clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+set -x
+
+apt-get -y autoremove
+rm -rf /var/lib/apt/lists/*

.github/workflows/build/scripts/user.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+set -x
+
+USERID="$1"
+USERNAME="$2"
+
+useradd -ms /bin/bash -u "$USERID" "$USERNAME"
+
+USERHOME=$(eval echo "~$USERNAME")
+VENVPATH="$USERHOME/$3"
+su -c "virtualenv -p python3.5 \"$VENVPATH\"" - "$USERNAME"
+
+# TODO virtualenv activation seems as better approach
+# but it's more tricky (failed to find out how) to automate
+# that for all cases (e.g. non interactive docker run/exec)
+USER_PYTHON=$(su -c "which python" - "$USERNAME")
+USER_PIP=$(su -c "which pip" - "$USERNAME")
+
+ln -sf "${VENVPATH}/bin/python" "$USER_PYTHON"
+ln -sf "${VENVPATH}/bin/pip" "$USER_PIP"

.github/workflows/lint/Dockerfile

@@ -0,0 +1,21 @@
+# Development
+FROM ubuntu:18.04
+LABEL maintainer="Kevin Griffin <griffin.kev@gmail.com>"
+
+RUN apt-get update && apt-get dist-upgrade -y
+
+# Install environment
+RUN apt-get install -y \
+	git \
+	wget \
+	python3.5 \
+	python3-pip \
+	python-setuptools \
+	python3-nacl
+
+RUN pip3 install -U \
+	'pip<10.0.0' \
+	setuptools \
+	pep8==1.7.1 \
+	pep8-naming==0.6.1 \
+	flake8==3.5.0

.github/workflows/lint/README.md

@@ -0,0 +1,15 @@
+## Indy Node build container
+This Docker container replaces the `code-validation.dockerfile` and is used by the GHA workflow for the lint job in Indy Node.
+
+## Managing this container
+
+```
+docker build .
+```
+
+```
+docker tag VERSION NAMESPACE/indy-node-build:TAG_NAME
+```
+
+```
+docker push NAMESPACE/indy-node-build:TAG_NAME

.gitignore

@@ -24,6 +24,9 @@ var/
 *.egg
 *.eggs
 
+# Needed for GitHub Actions
+!.github/workflows/build
+
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
@@ -72,8 +75,15 @@ include/
 # generated doc files
 docs/source/api_docs/
 
-# hidden files
-.*
+# IntelliJ specific config
+*.idea
+*.iml
+
+#vscode
+.vscode
 
 # Vagrant files
-.vagrant
+.vagrant
+
+# test output from working with GitHub actions
+test-result-node.xml

indy_common/test/auth/metadata/test_auth_rule_with_metadata_complex.py

@@ -1,3 +1,5 @@
+import pytest
+
 from indy_common.authorize.auth_constraints import AuthConstraint, IDENTITY_OWNER, AuthConstraintOr, AuthConstraintAnd, \
     AuthConstraintForbidden
 from indy_common.constants import ENDORSER
@@ -7,6 +9,7 @@
 MAX_SIG_COUNT = 3
 
 
+@pytest.mark.auth
 def test_plugin_and_or_rule_same_role_trustee_no_endorser(write_auth_req_validator, write_request_validation,
                                                           signatures, amount):
     validate(
@@ -40,6 +43,7 @@ def test_plugin_and_or_rule_same_role_trustee_no_endorser(write_auth_req_validat
     )
 
 
+@pytest.mark.auth
 def test_plugin_and_or_rule_diff_role_trustee_no_endorser(write_auth_req_validator, write_request_validation,
                                                           signatures, is_owner, amount):
     validate(
@@ -81,6 +85,7 @@ def test_plugin_and_or_rule_diff_role_trustee_no_endorser(write_auth_req_validat
     )
 
 
+@pytest.mark.auth
 def test_plugin_or_and_rule_diff_roles_trustee_no_endorser(write_auth_req_validator, write_request_validation,
                                                            signatures, is_owner, amount):
     validate(
@@ -128,6 +133,7 @@ def test_plugin_or_and_rule_diff_roles_trustee_no_endorser(write_auth_req_valida
     )
 
 
+@pytest.mark.auth
 def test_plugin_complex_trustee_no_endorser(write_auth_req_validator, write_request_validation,
                                             signatures, is_owner, amount):
     validate(
@@ -202,6 +208,7 @@ def test_plugin_complex_trustee_no_endorser(write_auth_req_validator, write_requ
     )
 
 
+@pytest.mark.auth
 def test_plugin_complex_with_and_rule_with_not_allowed(write_auth_req_validator, write_request_validation,
                                                        signatures, is_owner, off_ledger_signature, amount):
     validate(
@@ -219,6 +226,7 @@ def test_plugin_complex_with_and_rule_with_not_allowed(write_auth_req_validator,
     )
 
 
+@pytest.mark.auth
 def test_plugin_complex_with_or_rule_with_not_allowed_trustee_no_endorser(write_auth_req_validator,
                                                                           write_request_validation,
                                                                           signatures, is_owner, off_ledger_signature,