Merge branch 'develop' of github.com:gchq/gaffer-docker into gchqgh-34-…

…add-operation-declarations # Conflicts: # kubernetes/gaffer/templates/api/deployment.yaml # kubernetes/gaffer/values.yaml
macenturalxl1 · Jun 22, 2021 · 3818e9b · 3818e9b
2 parents dbb91ac + a0454b4
commit 3818e9b
Show file tree

Hide file tree

Showing 136 changed files with 2,820 additions and 1,127 deletions.
diff --git a/.github/release-notes.yml b/.github/release-notes.yml
@@ -0,0 +1,17 @@
+releasenotes:
+  sections:
+  - title: "Headliners"
+    emoji: ":star:"
+    labels: [ "headliner" ]
+  - title: "New Features"
+    emoji: ":sparkles:"
+    labels: [ "enhancement" ]
+  - title: "Bugs Fixed"
+    emoji: ":beetle:"
+    labels: [ "bug" ]
+  - title: "Documentation"
+    emoji: ":book:"
+    labels: "documentation"
+  - title: "Automation"
+    emoji: ":robot:"
+    labels: [ "automation" ]
diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml
@@ -0,0 +1,27 @@
+name: Continuous Integration
+
+on: [ pull_request ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Kubernetes
+      run: ./cd/before_install.sh
+
+    - name: Build Images
+      run: ./cd/build_images.sh
+
+    - name: Install subcharts
+      run: ./cd/install_dependencies.sh
+
+    - name: Lint charts
+      run: ./cd/lint_charts.sh
+
+    - name: Deploy to Kubernetes
+      run: ./cd/deploy_to_kind.sh
+
+    - name: Run Tests
+      run: helm test gaffer
diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml
@@ -0,0 +1,63 @@
+name: Integration Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 9 * * 1,4"
+jobs:
+  Build:
+    runs-on: ubuntu-latest
+    env:
+      GAFFER_VERSION: develop
+      GAFFER_TOOLS_VERSION: develop
+    steps:    
+    - uses: actions/checkout@v2
+
+    - name: Install KinD
+      run: ./cd/before_install.sh
+
+      # See issue #153 for why Gaffer image is not using docker-compose
+    - name: Build images
+      run: |
+        docker-compose --project-directory ./docker/gaffer-ui/ -f ./docker/gaffer-ui/docker-compose.yaml build --build-arg GAFFER_TOOLS_VERSION=$GAFFER_TOOLS_VERSION
+        docker build -t gchq/gaffer:$GAFFER_VERSION --build-arg GAFFER_VERSION=$GAFFER_VERSION ./docker/gaffer
+        docker build -t gchq/gaffer-integration-tests:$GAFFER_VERSION ./docker/gaffer-integration-tests
+    
+    - name: Install Helm charts
+      run: ./cd/install_dependencies.sh
+
+    - name: Start KinD
+      run: |
+        kind create cluster --config ./cd/kind.yaml
+        kind load docker-image gchq/gaffer:$GAFFER_VERSION
+        kind load docker-image gchq/gaffer-integration-tests:$GAFFER_VERSION
+      
+    - name: Deploy Gaffer
+      run: |
+        helm install gaffer ./kubernetes/gaffer --timeout=6m -f ./kubernetes/gaffer/integration-tests.yaml \
+          --set accumulo.hdfs.config.hdfsSite."dfs\.namenode\.datanode\.registration\.ip-hostname-check"=false \
+          --set accumulo.image.tag=develop
+
+    - name: Run Integration Tests
+      run: helm test gaffer --timeout=40m
+
+    - name: Send success message to ms teams
+      if: ${{ success()}}
+      uses: fjogeleit/http-request-action@master
+      with:
+          url: ${{ secrets.WORKFLOW_URL}}
+          method: 'POST'
+          data: '{"@context": "https://schema.org/extensions","@type": "MessageCard","title": "GitHub Actions Message","text": "Gaffer succesfully deployed and Integration Tests passing \n\n Click [here](https://github.com/gchq/gaffer-docker/actions) for workflow overview and logs"}'
+
+    - name: Print logs from Integration tests
+      if: ${{ failure() }}
+      run: kubectl logs pods/gaffer-accumulo-integration-tests
+
+    - name: Send failure message to ms teams
+      if: ${{ failure()}}
+      uses: fjogeleit/http-request-action@master
+      with:
+          url: ${{ secrets.WORKFLOW_URL}}
+          method: 'POST'
+          data: '{"@context": "https://schema.org/extensions","@type": "MessageCard","title": "GitHub Actions Message","text": "Gaffer Integration Tests Unsuccessful \n\n Click [here](https://github.com/gchq/gaffer-docker/actions) for workflow overview and logs"}'  
+
diff --git a/.github/workflows/link-issue.yaml b/.github/workflows/link-issue.yaml
@@ -0,0 +1,15 @@
+name: 'Issue Links'
+on:
+  pull_request:
+    types: [opened]
+
+jobs:
+  issue-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tkt-actions/add-issue-links@v1.4.0
+        with:
+          repo-token: '${{ secrets.GITHUB_TOKEN }}'
+          branch-prefix: 'gh-'
+          link-style: 'body'
+          resolve: 'true' 
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -0,0 +1,196 @@
+name: Release
+
+on:
+  milestone:
+    types:
+    - closed
+
+jobs:
+  create-git-tag:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: develop
+        token: ${{ secrets.ADMIN_GITHUB_TOKEN }}
+
+    - name: Set up Github credentials
+      run: |
+        git config user.name Gaffer
+        git config user.email github-actions@github.com
+    
+    - name: Set version
+      run: echo "RELEASE_VERSION=$(echo ${{ github.event.milestone.title }} | cut -c 2-)" >> $GITHUB_ENV
+
+    - name: Update version
+      run: ./cd/update_app_version.sh ${RELEASE_VERSION}
+
+    - name: Push changes
+      run: |
+        git commit -a -m "Release v${RELEASE_VERSION}"
+        git tag v${RELEASE_VERSION}
+        git push origin --tags
+        git push
+
+  update-branches:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        branch:
+        - master
+        - gh-pages
+    needs:
+    - create-git-tag
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ matrix.branch }}
+        token: ${{ secrets.ADMIN_GITHUB_TOKEN }}
+
+    - name: Set up Github credentials
+      run: |
+        git config user.name Gaffer
+        git config user.email github-actions@github.com
+  
+    - name: Update branch
+      run: |
+        git remote set-branches --add origin develop
+        git pull
+        git merge origin/develop --strategy recursive --strategy-option theirs --allow-unrelated-histories 
+
+    - name: Push changes
+      run: git push
+
+  publish-images-to-dockerhub:
+    runs-on: ubuntu-latest
+    needs:
+    - create-git-tag
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: develop
+
+    - name: Build images
+      run: ./cd/build_images.sh
+
+    - name: Set version
+      run: echo "APP_VERSION=$(echo ${{ github.event.milestone.title }} | cut -c 2-)" >> $GITHUB_ENV
+
+    - name: Publish images
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: ./cd/publish_images.sh
+
+  create-release-notes:
+    runs-on: ubuntu-latest
+    needs:
+    - create-git-tag
+    outputs:
+      release_upload_url: ${{ steps.upload_notes.outputs.upload_url }}
+    steps:
+    - name: Checkout develop
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+
+    - name: Create Release Notes
+      id: create_release_notes
+      uses: docker://decathlon/release-notes-generator-action:2.1.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set version
+      run: echo "RELEASE_VERSION=$(echo ${{ github.event.milestone.title }} | cut -c 2-)" >> $GITHUB_ENV
+
+    - name: Upload notes
+      id: upload_notes
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: v${{ env.RELEASE_VERSION }}
+        release_name: Gaffer Docker ${{ env.RELEASE_VERSION }}
+        body_path: release_file.md
+
+  upload-release-artifacts:
+    runs-on: ubuntu-latest
+    needs:
+    - create-release-notes
+    strategy:
+      matrix:
+        chart:
+        - hdfs
+        - accumulo
+        - gaffer
+        - gaffer-road-traffic
+    steps:
+    - name: Checkout develop
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+
+    - name: Install helm
+      run: ./cd/before_install.sh
+
+    - name: Install dependencies
+      run: ./cd/install_dependencies.sh
+
+    - name: Create Asset
+      env:
+        chart: ${{ matrix.chart }}
+      run: |
+        helm package "kubernetes/${chart}"
+        filename=$(ls | grep ${chart}-[0-9]*.[0-9]*.[0-9]*.tgz)
+        echo "ASSET_FILENAME=${filename}" >> $GITHUB_ENV
+    
+    - name: Upload chart artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: ${{ matrix.chart }}
+        path: ${{ env.ASSET_FILENAME }}
+        retention-days: 1
+
+    - name: Upload Asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create-release-notes.outputs.release_upload_url }}
+        asset_path: ${{ env.ASSET_FILENAME }}
+        asset_name: ${{ env.ASSET_FILENAME }}
+        asset_content_type: application/zip
+
+  update-helm-repo:
+    runs-on: ubuntu-latest
+    needs:
+    - upload-release-artifacts
+    steps:
+    - name: Checkout gh-pages
+      uses: actions/checkout@v2
+      with:
+        ref: gh-pages
+        token: ${{ secrets.ADMIN_GITHUB_TOKEN }}
+
+    - name: Set up Github credentials
+      run: |
+        git config user.name Gaffer
+        git config user.email github-actions@github.com
+
+    - name: Install helm
+      run: ./cd/before_install.sh
+
+    - name: Download packaged charts
+      uses: actions/download-artifact@v2
+      with:
+        path: artifacts
+
+    - name: Update repo
+      run: |
+        mv artifacts/*/*.tgz .
+        helm repo index . --url "https://github.com/gchq/gaffer-docker/releases/download/${{ github.event.milestone.title }}" --merge index.yaml
+        sed -i'' -e 's|file://.*|https://gchq.github.io/gaffer-docker|g' index.yaml
+        rm *.tgz
+        rm -r artifacts
+        git commit -am "Updated index.yaml"
+        git push
diff --git a/README.md b/README.md
@@ -11,26 +11,23 @@ This repo contains Dockerfiles for building container images for:
 
 It also contains Helm Charts so that the following applications can be deployed onto Kubernetes clusters:
 * [HDFS](kubernetes/hdfs/)
+* [Accumulo](kubernetes/accumulo/)
 * [Gaffer](kubernetes/gaffer/)
 * [Example Gaffer Graph containing Road Traffic Dataset](kubernetes/gaffer-road-traffic/)
 
 These charts can be accessed by cloning our repository or by using our Helm repo hosted on our [Github Pages Site](https://gchq.github.io/gaffer-docker)
 
 To use this repo, run:
 ```bash
-helm repo add gaffer https://gchq.github.io/gaffer-docker
+helm repo add gaffer-docker https://gchq.github.io/gaffer-docker
 ```
 
-There are guides on how to deploy the charts on:
-* a local Kubernetes cluster, [using kind (Kubernetes IN Docker)](kubernetes/kind-deployment.md)
-* an [AWS EKS cluster](kubernetes/aws-eks-deployment.md)
+## Kubernetes How-to Guides
 
-These guides assume you've cloned the repository and are using the Helm charts contained within. We do also host our Helm Charts index on our Github pages site. You can get access to them by running:
-```bash
-helm repo add gaffer https://gchq.github.io/gaffer-docker
-```
+We have a number of [guides](./kubernetes/docs/guides.md) to help you deploy Gaffer on Kubernetes.
 
 ## Versioning
+
 Each of our images will be tagged in DockerHub with the version of the software they represent. Every release,
 we update the image for that tag and add a new release which has the corresponding git tag.
 
@@ -47,7 +44,11 @@ Note that we maintain mutable versions of latest, as well as the major, minor an
 ensure that your image will never change when doing a pull from docker, make sure to use the version with the git tag in the
 build metadata.
 
-This process is automated by Travis CI.
+This process is automated by Github actions.
+
+## Known Compatible Docker Versions
+
+* 20.10.5
 
 ## Contributing
 

diff --git a/app_version b/app_version
diff --git a/cd/before_install.sh b/cd/before_install.sh
@@ -16,21 +16,20 @@
 
 set -e
 
+KUBERNETES_VERSION=`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`
+
 # Install Kubectl
 curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
 chmod +x kubectl
 sudo mv kubectl /usr/local/bin/
 
 # Install Helm
-curl -Lo ./helm.tar.gz https://get.helm.sh/helm-v3.1.1-linux-amd64.tar.gz
+curl -Lo ./helm.tar.gz https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz
 tar -zxvf helm.tar.gz
 sudo mv linux-amd64/helm /usr/local/bin/
 
-if [ ${TRAVIS_PULL_REQUEST} == 'false' ]; then
-    exit 0
-fi
 
 # Install Kind
-curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(uname)-amd64
+curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.10.0/kind-$(uname)-amd64
 chmod +x ./kind
 sudo mv ./kind /usr/local/bin/kind