diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3dc4d3e85..05c141f22 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,29 +1,13 @@ # GitHub Actions Workflow -The workflow in the [build.yaml](build.yaml) file replaces the existing [Jenkins.ci](../../Jenkinsfile.ci) build process. - -The `lint` job replaces the `Static code validation` stage of the Jenkins pipeline, while the remainder of the jobs replace the `Build / Test` stage. - -The `Build result notification` stage was not moved to GHA, as build failures will be reports via GHA. - -Many of the other stages are replaced merely by the fact we're using Github Actions, we use prebuild Docker containers so we don't have to replicate the steps for building containers. - -The build process for `Jenkins.nightly` was not ported to GHA. - -Support for Windows continues as a `ToDo` item. - - -## Configuring actions - -If you are cloning or forking this repo you will need to configure two secrets for Actions to run correctly. - -Secrets can be set via Settings -> Secrets -> New repository secret: - -`CR_USER`: is your GH username. It must be lowercase. -`CR_PAT`: can be created by following the [Creating a personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) documentation. - -When you create your token, the only permission you need to select is `write:packages` **Upload packages to GitHub package registry**, all other necessary permissions will be selected by default. - -You may also need to enable [Improved container support](https://docs.github.com/en/packages/guides/enabling-improved-container-support) in order to allow the images to be written to your repository. You'll see an error to this affect if this is the case. - -Once you have run the build once with those secrets, you have to make the images public. Access the packages at https://ghcr.io/USER/indy-node/node-build and https://ghcr.io/USER/indy-node/node-lint and change the visibility in 'Package Settings' to 'Public' then re-run the build. Alternatively, if you would prefer to keep the images private, you can manage access to the package and select only the user account associated with the token you setup above. \ No newline at end of file +The workflow in the [push_pr.yaml](push_pr.yaml) file runs on push and pull requests to the ubuntu-20-04-upgrade branch. +It uses the following reusable workflows in this folder. + ++ [buildimage.yaml](buildimage.yaml) + This workflow builds the dockerimages and pushes them to the GHCR. ++ [test.yaml](test.yaml) + This workflow runs the tests inside the uploaded docker images. ++ [buildpackages.yaml](buildpackages.yaml) + This workflows builds the python and debian packages. It also uploads them to the workflow. ++ [publish_artifacts.yaml](publish_artifacts.yaml) + This workflow uploads the packages to PYPI and Artifactory. \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 275ccdb2d..000000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,398 +0,0 @@ -name: indy-node-build -on: - push: - branches: - - ubuntu-20.04-upgrade - - pull_request: - branches: - - ubuntu-20.04-upgrade - workflow_dispatch: - -jobs: - workflow-setup: - name: Initialize Workflow - runs-on: ubuntu-latest - outputs: - CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }} - CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }} - UBUNTU_VERSION: ${{ steps.cache.outputs.UBUNTU_VERSION }} - # Expose the lowercase version of the GitHub repository name - # to all subsequent jobs that reference image repositories - # as the push and pull operations require the URL of the repository - # to be in lowercase. - GITHUB_REPOSITORY_NAME: ${{ steps.repository-name.outputs.lowercase }} - GITHUB_REF: ${{ steps.cache.outputs.GITHUB_REF }} - distribution: ${{ steps.cache.outputs.distribution }} - isDev: ${{ steps.build-flags.outputs.isDev }} - isRC: ${{ steps.build-flags.outputs.isRC }} - publish: ${{ steps.build-flags.outputs.publish }} - steps: - - name: Git checkout - uses: actions/checkout@v2 - - - name: Convert the GitHub repository name to lowercase - id: repository-name - uses: ASzc/change-string-case-action@v1 - with: - string: ${{ github.repository }} - - - name: Set outputs - id: cache - run: | - # Set variables according to version of ubuntu - if [[ "${{github.base_ref}}" == "master" || "${{github.ref}}" == "refs/heads/master" ]]; then - echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile.ubuntu-1604') }}" - echo "::set-output name=UBUNTU_VERSION::ubuntu-1604" - echo "::set-output name=distribution::xenial" - fi - if [[ "${{github.base_ref}}" == "ubuntu-20.04-upgrade" || "${{github.ref}}" == "refs/heads/ubuntu-20.04-upgrade" ]]; then - echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile.ubuntu-2004') }}" - echo "::set-output name=UBUNTU_VERSION::ubuntu-2004" - echo "::set-output name=distribution::focal" - fi - - - if [[ "${{github.base_ref}}" == 'master' || "${{github.ref}}" == 'refs/heads/master' || "${{github.base_ref}}" == 'main' || "${{github.ref}}" == 'refs/heads/main' ]]; then - echo "::set-output name=GITHUB_REF::main" - elif [[ "${{github.base_ref}}" == 'release*' || "${{github.ref}}" == 'refs/heads/release*' ]]; then - echo "::set-output name=GITHUB_REF::rc" - elif [[ "${{github.base_ref}}" == 'stable' || "${{github.ref}}" == 'refs/heads/stable' ]]; then - echo "::set-output name=GITHUB_REF::stable" - else - echo "::set-output name=GITHUB_REF::dev" - fi - - - name: Set build flags - id: build-flags - run: | - - if [[ "${{steps.cache.outputs.GITHUB_REF}}" == 'dev' || "${{steps.cache.outputs.GITHUB_REF}}" == 'main' ]]; then - echo "::set-output name=isDev::true" - else - echo "::set-output name=isDev::false" - fi - - if [[ "${{steps.cache.outputs.GITHUB_REF}}" == 'rc' ]]; then - echo "::set-output name=isRC::true" - else - echo "::set-output name=isRC::false" - fi - - # Ensure publishing is only performed when the build is executed from the main (hyperledger/indy-node) repository. - if [[ ${{github.event.repository.full_name}} == 'hyperledger/indy-node' && ${{github.event_name}} == 'push' && ( ${{steps.cache.outputs.GITHUB_REF}} == 'main' || ${{steps.cache.outputs.GITHUB_REF}} == 'rc' || ${{steps.cache.outputs.GITHUB_REF}} == 'stable' || ${{steps.cache.outputs.GITHUB_REF}} == 'dev' ) ]]; then - echo "::set-output name=publish::true" - else - echo "::set-output name=publish::false" - fi - - build-image: - name: Create Builder Image - # Reference to workflow-setup job is required to access its various outputs. - needs: workflow-setup - runs-on: ubuntu-latest - env: - CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} - GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} - UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - steps: - - name: Git checkout - uses: actions/checkout@v2 - - - name: Try load from cache - id: cache-image - uses: actions/cache@v2 - with: - path: ${GITHUB_WORKSPACE}/cache - key: ${{ env.CACHE_KEY_BUILD}} - - - name: Prepare image labels and tags - if: steps.cache-image.outputs.cache-hit != 'true' - id: prep - shell: bash - run: | - DOCKER_IMAGE=ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-build - TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${{ env.UBUNTU_VERSION }}" - echo ::set-output name=tags::${TAGS} - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - - - name: Log into the GitHub Container Registry - if: steps.cache-image.outputs.cache-hit != 'true' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - if: steps.cache-image.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@v1 - - - name: Build and push image - if: steps.cache-image.outputs.cache-hit != 'true' - uses: docker/build-push-action@v2 - with: - context: . - file: .github/workflows/build/Dockerfile.${{ env.UBUNTU_VERSION }} - no-cache: true - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - - - name: Touch Cache - if: steps.cache-image.outputs.cache-hit != 'true' - run: | - mkdir -p ${GITHUB_WORKSPACE}/cache - touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }} - - - - indy_node_tests: - name: Sliced Module Tests - # Reference to workflow-setup job is required to access the GITHUB_REPOSITORY_NAME output. - needs: [workflow-setup, build-image] - runs-on: ubuntu-20.04 - # Fix for scacap/action-surefire-report out of memory error: - # - https://github.com/ScaCap/action-surefire-report/issues/17 - env: - NODE_OPTIONS: '--max_old_space_size=4096' - #SLICE_TOTAL_SLICES needs to match the total number of slices in the matrix strategy. - SLICE_TOTAL_SLICES: 11 - container: - image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - strategy: - matrix: - module: [indy_node, indy_common] - slice: [1, 2, 3, 4 ,5, 6, 7, 8, 9, 10, 11] - fail-fast: false - steps: - - name: Check out code - uses: actions/checkout@v2 - - # =============================================== - # Caching cannot be used. - # - For some reason as soon as it is enabled - # the test start complaining about zmq missing - # for the plenum install. - # ----------------------------------------------- - # - name: Cache pip - # uses: actions/cache@v2 - # with: - # # pip cache on the node-build image is not in the default location. - # # path: ~/.cache/pip - # path: /root/.cache/pip - # key: ${{ runner.os }}-indy-node-pip-${{ hashFiles('**/requirements.txt', '**/setup.py') }} - # restore-keys: | - # ${{ runner.os }}-indy-node-pip- - - - name: Install dependencies - run: | - # Explicitly use the existing pip cache location in the node-build image. - pip --cache-dir /root/.cache/pip install .[tests] - - - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/ ${{ env.SLICE_TOTAL_SLICES }} - id: node-test - run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv" --dir "${{ matrix.module }}" --output "test-result-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/ ${{ env.SLICE_TOTAL_SLICES }}" - - - name: Publish Test Report - if: success() || failure() - uses: scacap/action-surefire-report@v1.0.7 - continue-on-error: true - with: - check_name: Indy Node ${{ matrix.module }} Test Report for slice ${{ matrix.slice }}/${{ strategy.job-total }} - github_token: ${{ secrets.GITHUB_TOKEN }} - report_paths: "*-test-results.xml" - - - name: Upload Detailed Test Failure Results - # The test runner only emits the detailed test results if the tests fail. - if: (steps.node-test.outcome == 'failure') && failure() - uses: actions/upload-artifact@v2 - with: - name: detailed-test-result-slice-${{ matrix.slice }} - path: test-result-node-${{ matrix.slice }}.txt - retention-days: 5 - - lint: - name: Lint - # Reference to workflow-setup job is required to access the GITHUB_REPOSITORY_NAME output. - needs: [workflow-setup] - runs-on: ubuntu-20.04 - steps: - - name: Check out code - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-lint - restore-keys: | - ${{ runner.os }}-pip-lint - - name: Install flake8 - run: pip install flake8==3.8.4 pep8==1.7.1 pep8-naming==0.6.1 - - name: Lint with flake8 - run: python3 -m flake8 . - - build_release: - name: Build Release - needs: [workflow-setup, indy_node_tests, lint] - runs-on: ubuntu-20.04 - env: - UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - container: - image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - steps: - - name: Check out code - uses: actions/checkout@v1 - - - name: Set Build Version - id: version - uses: ./.github/actions/set-version - with: - moduleName: indy_node - isDev: ${{ needs.workflow-setup.outputs.isDev }} - isRC: ${{ needs.workflow-setup.outputs.isRC }} - - - name: Build Deployment Package - run: | - mkdir -p /tmp/node-build - ./build-scripts/${{ env.UBUNTU_VERSION }}/build-indy-node.sh "/__w/indy-node/indy-node" "${{ steps.version.outputs.upstreamVer }}" "/tmp/node-build" "${{ steps.version.outputs.pkgVer }}" - - - uses: actions/upload-artifact@v2 - with: - name: node-deb - path: /tmp/node-build - retention-days: 5 - - build_3rd_party_dependencies: - name: Build 3rd Party Dependencies - needs: [workflow-setup, indy_node_tests, lint] - runs-on: ubuntu-20.04 - env: - UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - container: - image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - steps: - - name: Check out code - uses: actions/checkout@v1 - - - name: Try load from cache. - id: third-party-dependencies - uses: actions/cache@v2 - with: - path: /tmp/third-party-dependencies - key: ${{ format('third-party-dependencies-{0}', hashFiles(format('./build-scripts/{0}/build-3rd-parties.sh', needs.workflow-setup.outputs.UBUNTU_VERSION ))) }} - - - name: Build 3rd party deployment packages - if: steps.third-party-dependencies.outputs.cache-hit != 'true' - run: | - mkdir -p ./build-scripts/${{ env.UBUNTU_VERSION }}/cache/3rd-party-dependencies/ - ./build-scripts/${{ env.UBUNTU_VERSION }}/build-3rd-parties.sh ./cache/3rd-party-dependencies - mv ./build-scripts/${{ env.UBUNTU_VERSION }}/cache/* /tmp/third-party-dependencies - - build-python-packages: - name: Build Python Packages - runs-on: ubuntu-20.04 - needs: [workflow-setup, indy_node_tests, lint] - steps: - - name: Check out code - uses: actions/checkout@v1 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install required packages via pip - run: | - python3 -m pip install pytest-runner wheel - - - name: Set Build Version - id: version - uses: ./.github/actions/set-version - with: - moduleName: indy_node - isDev: ${{ needs.workflow-setup.outputs.isDev }} - isRC: ${{ needs.workflow-setup.outputs.isRC }} - - - name: Prepare package and set version - run: | - ./build-scripts/${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}/prepare-package.sh . indy_node "${{ steps.version.outputs.upstreamVer }}" python-packages - - - name: Building python package - run: | - python3 setup.py sdist --dist-dir /tmp/dist bdist_wheel --dist-dir /tmp/dist - - - uses: actions/upload-artifact@v2 - with: - name: node-python - path: /tmp/dist - retention-days: 5 - - publish_artifacts: - name: Publish Artifacts - runs-on: ubuntu-20.04 - needs: [workflow-setup, build_release, build_3rd_party_dependencies, build-python-packages] - if: needs.workflow-setup.outputs.publish == 'true' - env: - GITHUB_REF: ${{ needs.workflow-setup.outputs.GITHUB_REF }} - distribution: ${{ needs.workflow-setup.outputs.distribution }} - steps: - - name: Check out code - uses: actions/checkout@v1 - - - name: Setup JFrog CLI - uses: jfrog/setup-jfrog-cli@v2 - env: - JF_ARTIFACTORY_1: ${{ secrets.INDY_ARTIFACTORY_REPO_CONFIG }} - - - name: Ping Artifactory - run: | - # Test the connection to Ping the Hyperledger Artifactory server - # to ensure everything has been setup correctly. - jfrog rt ping - - - name: Download Node Artifacts from Pipeline Artifacts - uses: actions/download-artifact@v2 - with: - name: node-deb - path: to_publish - - - name: Publish Node Artifacts - uses: ./.github/actions/publish-deb - with: - sourceDirectory: /home/runner/work/indy-node/indy-node/to_publish - distribution: ${{ env.distribution }} - component: ${{ env.GITHUB_REF }} - - - name: Download 3rd Party Artifacts Dependencies from Cache - id: third-party-dependencies - uses: actions/cache@v2 - with: - path: /tmp/third-party-dependencies - key: ${{ format('third-party-dependencies-{0}', hashFiles(format('./build-scripts/{0}/build-3rd-parties.sh', needs.workflow-setup.outputs.UBUNTU_VERSION ))) }} - - - name: Publish 3rd Party Dependencies - uses: ./.github/actions/publish-deb - with: - sourceDirectory: /home/runner/tmp/third-party-dependencies - distribution: ${{ env.distribution }} - component: ${{ env.GITHUB_REF }} - - - name: Download Python Packages from Pipeline Artifacts - uses: actions/download-artifact@v2 - with: - name: node-python - path: dist - - - name: Publish Python Package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - skip_existing: true \ No newline at end of file diff --git a/.github/workflows/buildimage.yaml b/.github/workflows/buildimage.yaml new file mode 100644 index 000000000..9b4b110c2 --- /dev/null +++ b/.github/workflows/buildimage.yaml @@ -0,0 +1,75 @@ +name: "Build Docker Image" + +on: + workflow_call: + inputs: + CACHE_KEY_BUILD: + required: true + type: string + GITHUB_REPOSITORY_NAME: + required: true + type: string + UBUNTU_VERSION: + required: true + type: string + +jobs: + build-image: + name: Create Builder Image + runs-on: ubuntu-latest + env: + CACHE_KEY_BUILD: ${{ inputs.CACHE_KEY_BUILD }} + GITHUB_REPOSITORY_NAME: ${{ inputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ inputs.UBUNTU_VERSION }} + steps: + - name: Git checkout + uses: actions/checkout@v2 + + - name: Try load from cache + id: cache-image + uses: actions/cache@v2 + with: + path: ${GITHUB_WORKSPACE}/cache + key: ${{ env.CACHE_KEY_BUILD}} + + - name: Prepare image labels and tags + if: steps.cache-image.outputs.cache-hit != 'true' + id: prep + shell: bash + run: | + DOCKER_IMAGE=ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-build + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${{ env.UBUNTU_VERSION }}" + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + + - name: Log into the GitHub Container Registry + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Build and push image + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: .github/workflows/build/Dockerfile.${{ env.UBUNTU_VERSION }} + no-cache: true + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Touch Cache + if: steps.cache-image.outputs.cache-hit != 'true' + run: | + mkdir -p ${GITHUB_WORKSPACE}/cache + touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }} \ No newline at end of file diff --git a/.github/workflows/buildpackages.yaml b/.github/workflows/buildpackages.yaml new file mode 100644 index 000000000..a5e82d926 --- /dev/null +++ b/.github/workflows/buildpackages.yaml @@ -0,0 +1,111 @@ +name: "Build Node Packages" + +on: + workflow_call: + inputs: + GITHUB_REPOSITORY_NAME: + required: true + type: string + UBUNTU_VERSION: + required: true + type: string + isDev: + required: true + type: string + isRC: + required: true + type: string + +jobs: + build_release: + name: Build Release + runs-on: ubuntu-20.04 + env: + UBUNTU_VERSION: ${{ inputs.UBUNTU_VERSION }} + container: + image: ghcr.io/${{ inputs.GITHUB_REPOSITORY_NAME }}/node-build:${{ inputs.UBUNTU_VERSION }} + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Set Build Version + id: version + uses: ./.github/actions/set-version + with: + moduleName: indy_node + isDev: ${{ inputs.isDev }} + isRC: ${{ inputs.isRC }} + + - name: Build Deployment Package + run: | + mkdir -p /tmp/node-build + ./build-scripts/${{ env.UBUNTU_VERSION }}/build-indy-node.sh "/__w/indy-node/indy-node" "${{ steps.version.outputs.upstreamVer }}" "/tmp/node-build" "${{ steps.version.outputs.pkgVer }}" + + - uses: actions/upload-artifact@v2 + with: + name: node-deb + path: /tmp/node-build + retention-days: 5 + + build_3rd_party_dependencies: + name: Build 3rd Party Dependencies + runs-on: ubuntu-20.04 + env: + UBUNTU_VERSION: ${{ inputs.UBUNTU_VERSION }} + container: + image: ghcr.io/${{ inputs.GITHUB_REPOSITORY_NAME }}/node-build:${{ inputs.UBUNTU_VERSION }} + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Try load from cache. + id: third-party-dependencies + uses: actions/cache@v2 + with: + path: /tmp/third-party-dependencies + key: ${{ format('third-party-dependencies-{0}', hashFiles(format('./build-scripts/{0}/build-3rd-parties.sh', inputs.UBUNTU_VERSION ))) }} + + - name: Build 3rd party deployment packages + if: steps.third-party-dependencies.outputs.cache-hit != 'true' + run: | + mkdir -p ./build-scripts/${{ env.UBUNTU_VERSION }}/cache/3rd-party-dependencies/ + ./build-scripts/${{ env.UBUNTU_VERSION }}/build-3rd-parties.sh ./cache/3rd-party-dependencies + mv ./build-scripts/${{ env.UBUNTU_VERSION }}/cache/* /tmp/third-party-dependencies + + build-python-packages: + name: Build Python Packages + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install required packages via pip + run: | + python3 -m pip install pytest-runner wheel + + - name: Set Build Version + id: version + uses: ./.github/actions/set-version + with: + moduleName: indy_node + isDev: ${{ inputs.isDev }} + isRC: ${{ inputs.isRC }} + + - name: Prepare package and set version + run: | + ./build-scripts/${{ inputs.UBUNTU_VERSION }}/prepare-package.sh . indy_node "${{ steps.version.outputs.upstreamVer }}" python-packages + + - name: Building python package + run: | + python3 setup.py sdist --dist-dir /tmp/dist bdist_wheel --dist-dir /tmp/dist + + - uses: actions/upload-artifact@v2 + with: + name: node-python + path: /tmp/dist + retention-days: 5 diff --git a/.github/workflows/publish_artifacts.yaml b/.github/workflows/publish_artifacts.yaml new file mode 100644 index 000000000..cb7aa2c4c --- /dev/null +++ b/.github/workflows/publish_artifacts.yaml @@ -0,0 +1,83 @@ +name: "Publish Artifacts" + +on: + workflow_call: + inputs: + GITHUB_REF: + required: true + type: string + UBUNTU_VERSION: + required: true + type: string + distribution: + required: true + type: string + secrets: + INDY_ARTIFACTORY_REPO_CONFIG: + required: true + PYPI_API_TOKEN: + required: true + + +jobs: + publish_artifacts: + name: Publish Artifacts + runs-on: ubuntu-20.04 + env: + GITHUB_REF: ${{ inputs.GITHUB_REF }} + distribution: ${{ inputs.distribution }} + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Setup JFrog CLI + uses: jfrog/setup-jfrog-cli@v2 + env: + JF_ARTIFACTORY_1: ${{ secrets.INDY_ARTIFACTORY_REPO_CONFIG }} + + - name: Ping Artifactory + run: | + # Test the connection to Ping the Hyperledger Artifactory server + # to ensure everything has been setup correctly. + jfrog rt ping + + - name: Download Node Artifacts from Pipeline Artifacts + uses: actions/download-artifact@v2 + with: + name: node-deb + path: to_publish + + - name: Publish Node Artifacts + uses: ./.github/actions/publish-deb + with: + sourceDirectory: /home/runner/work/indy-node/indy-node/to_publish + distribution: ${{ env.distribution }} + component: ${{ env.GITHUB_REF }} + + - name: Download 3rd Party Artifacts Dependencies from Cache + id: third-party-dependencies + uses: actions/cache@v2 + with: + path: /tmp/third-party-dependencies + key: ${{ format('third-party-dependencies-{0}', hashFiles(format('./build-scripts/{0}/build-3rd-parties.sh', inputs.UBUNTU_VERSION ))) }} + + - name: Publish 3rd Party Dependencies + uses: ./.github/actions/publish-deb + with: + sourceDirectory: /home/runner/tmp/third-party-dependencies + distribution: ${{ env.distribution }} + component: ${{ env.GITHUB_REF }} + + - name: Download Python Packages from Pipeline Artifacts + uses: actions/download-artifact@v2 + with: + name: node-python + path: dist + + - name: Publish Python Package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + skip_existing: true + \ No newline at end of file diff --git a/.github/workflows/push_pr.yaml b/.github/workflows/push_pr.yaml new file mode 100644 index 000000000..cd630ce32 --- /dev/null +++ b/.github/workflows/push_pr.yaml @@ -0,0 +1,149 @@ +name: Indy Node - Push and PR Workflow +on: + push: + branches: + - ubuntu-20.04-upgrade + + pull_request: + branches: + - ubuntu-20.04-upgrade + workflow_dispatch: + +jobs: + workflow-setup: + name: Initialize Workflow + runs-on: ubuntu-latest + outputs: + CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }} + UBUNTU_VERSION: ${{ steps.cache.outputs.UBUNTU_VERSION }} + # Expose the lowercase version of the GitHub repository name + # to all subsequent jobs that reference image repositories + # as the push and pull operations require the URL of the repository + # to be in lowercase. + GITHUB_REPOSITORY_NAME: ${{ steps.repository-name.outputs.lowercase }} + GITHUB_REF: ${{ steps.cache.outputs.GITHUB_REF }} + distribution: ${{ steps.cache.outputs.distribution }} + isDev: ${{ steps.build-flags.outputs.isDev }} + isRC: ${{ steps.build-flags.outputs.isRC }} + publish: ${{ steps.build-flags.outputs.publish }} + steps: + - name: Git checkout + uses: actions/checkout@v2 + + - name: Convert the GitHub repository name to lowercase + id: repository-name + uses: ASzc/change-string-case-action@v1 + with: + string: ${{ github.repository }} + + - name: Set outputs + id: cache + run: | + # Set variables according to version of ubuntu + if [[ "${{github.base_ref}}" == "master" || "${{github.ref}}" == "refs/heads/master" ]]; then + echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile.ubuntu-1604') }}" + echo "::set-output name=UBUNTU_VERSION::ubuntu-1604" + echo "::set-output name=distribution::xenial" + fi + if [[ "${{github.base_ref}}" == "ubuntu-20.04-upgrade" || "${{github.ref}}" == "refs/heads/ubuntu-20.04-upgrade" ]]; then + echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile.ubuntu-2004') }}" + echo "::set-output name=UBUNTU_VERSION::ubuntu-2004" + echo "::set-output name=distribution::focal" + fi + + + if [[ "${{github.base_ref}}" == 'master' || "${{github.ref}}" == 'refs/heads/master' || "${{github.base_ref}}" == 'main' || "${{github.ref}}" == 'refs/heads/main' ]]; then + echo "::set-output name=GITHUB_REF::main" + elif [[ "${{github.base_ref}}" == 'release*' || "${{github.ref}}" == 'refs/heads/release*' ]]; then + echo "::set-output name=GITHUB_REF::rc" + elif [[ "${{github.base_ref}}" == 'stable' || "${{github.ref}}" == 'refs/heads/stable' ]]; then + echo "::set-output name=GITHUB_REF::stable" + else + echo "::set-output name=GITHUB_REF::dev" + fi + + - name: Set build flags + id: build-flags + run: | + + if [[ "${{steps.cache.outputs.GITHUB_REF}}" == 'dev' || "${{steps.cache.outputs.GITHUB_REF}}" == 'main' ]]; then + echo "::set-output name=isDev::true" + else + echo "::set-output name=isDev::false" + fi + + if [[ "${{steps.cache.outputs.GITHUB_REF}}" == 'rc' ]]; then + echo "::set-output name=isRC::true" + else + echo "::set-output name=isRC::false" + fi + + # Ensure publishing is only performed when the build is executed from the main (hyperledger/indy-node) repository. + if [[ ${{github.event.repository.full_name}} == 'hyperledger/indy-node' && ${{github.event_name}} == 'push' && ( ${{steps.cache.outputs.GITHUB_REF}} == 'main' || ${{steps.cache.outputs.GITHUB_REF}} == 'rc' || ${{steps.cache.outputs.GITHUB_REF}} == 'stable' || ${{steps.cache.outputs.GITHUB_REF}} == 'dev' ) ]]; then + echo "::set-output name=publish::true" + else + echo "::set-output name=publish::false" + fi + + lint: + name: Lint + # Reference to workflow-setup job is required to access the GITHUB_REPOSITORY_NAME output. + needs: [workflow-setup] + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-lint + restore-keys: | + ${{ runner.os }}-pip-lint + - name: Install flake8 + run: pip install flake8==3.8.4 pep8==1.7.1 pep8-naming==0.6.1 + - name: Lint with flake8 + run: python3 -m flake8 . + + build-image: + name: Create Builder Image + needs: [workflow-setup, lint] + uses: ./.github/workflows/buildimage.yaml + with: + CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} + + indy_node_tests: + name: Indy Node Tests + needs: [workflow-setup, build-image] + uses: ./.github/workflows/test.yaml + with: + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} + + build_packages: + name: Build Packages + needs: [workflow-setup, indy_node_tests] + uses: ./.github/workflows/buildpackages.yaml + with: + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} + isDev: ${{ needs.workflow-setup.outputs.isDev }} + isRC: ${{ needs.workflow-setup.outputs.isRC }} + + publish_artifacts: + name: Publish Artifacts + needs: [workflow-setup, build_packages] + if: needs.workflow-setup.outputs.publish == 'true' + uses: ./.github/workflows/publish_artifacts.yaml + with: + GITHUB_REF: ${{ needs.workflow-setup.outputs.GITHUB_REF }} + UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} + distribution: ${{ needs.workflow-setup.outputs.distribution }} + secrets: + INDY_ARTIFACTORY_REPO_CONFIG: ${{ secrets.INDY_ARTIFACTORY_REPO_CONFIG }} + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..46e3e5159 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,75 @@ +name: "Test Indy Node" + +on: + workflow_call: + inputs: + GITHUB_REPOSITORY_NAME: + required: true + type: string + UBUNTU_VERSION: + required: true + type: string + +jobs: + indy_node_tests: + name: Sliced Module Tests + runs-on: ubuntu-20.04 + # Fix for scacap/action-surefire-report out of memory error: + # - https://github.com/ScaCap/action-surefire-report/issues/17 + env: + NODE_OPTIONS: '--max_old_space_size=4096' + #SLICE_TOTAL_SLICES needs to match the total number of slices in the matrix strategy. + SLICE_TOTAL_SLICES: 11 + container: + image: ghcr.io/${{ inputs.GITHUB_REPOSITORY_NAME }}/node-build:${{ inputs.UBUNTU_VERSION }} + strategy: + matrix: + module: [indy_node, indy_common] + slice: [1, 2, 3, 4 ,5, 6, 7, 8, 9, 10, 11] + fail-fast: false + steps: + - name: Check out code + uses: actions/checkout@v2 + + # =============================================== + # Caching cannot be used. + # - For some reason as soon as it is enabled + # the test start complaining about zmq missing + # for the plenum install. + # ----------------------------------------------- + # - name: Cache pip + # uses: actions/cache@v2 + # with: + # # pip cache on the node-build image is not in the default location. + # # path: ~/.cache/pip + # path: /root/.cache/pip + # key: ${{ runner.os }}-indy-node-pip-${{ hashFiles('**/requirements.txt', '**/setup.py') }} + # restore-keys: | + # ${{ runner.os }}-indy-node-pip- + + - name: Install dependencies + run: | + # Explicitly use the existing pip cache location in the node-build image. + pip --cache-dir /root/.cache/pip install .[tests] + + - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/ ${{ env.SLICE_TOTAL_SLICES }} + id: node-test + run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv" --dir "${{ matrix.module }}" --output "test-result-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/ ${{ env.SLICE_TOTAL_SLICES }}" + + - name: Publish Test Report + if: success() || failure() + uses: scacap/action-surefire-report@v1.0.7 + continue-on-error: true + with: + check_name: Indy Node ${{ matrix.module }} Test Report for slice ${{ matrix.slice }}/${{ strategy.job-total }} + github_token: ${{ secrets.GITHUB_TOKEN }} + report_paths: "*-test-results.xml" + + - name: Upload Detailed Test Failure Results + # The test runner only emits the detailed test results if the tests fail. + if: (steps.node-test.outcome == 'failure') && failure() + uses: actions/upload-artifact@v2 + with: + name: detailed-test-result-slice-${{ matrix.slice }} + path: test-result-node-${{ matrix.slice }}.txt + retention-days: 5