Use magda common

.github/workflows/main.yml

@@ -11,37 +11,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 10
+      - name: Use Node.js 12
         uses: actions/setup-node@v1
         with:
-          node-version: 10
+          node-version: 12
       - run: yarn install
       - run: yarn build
       - run: yarn test
-
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.2.0
+
+      - name: helm-build-dependencies
+        run: yarn update-all-charts
 
       - run: yarn helm-lint
 
       - name: Login to GitHub Package Repository
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
+          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
         run: docker login docker.pkg.github.com -u magdabot -p ${GH_TOKEN}
       - name: Build Docker Image & Push
         run: yarn docker-build-prod --repository=docker.pkg.github.com/magda-io/${REPO_NAME} --name=${REPO_NAME} --version=${GITHUB_SHA}
-
-  build-test-node-12:
-    name: Build, Test Only with Node 12
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js 12
-        uses: actions/setup-node@v1
-        with:
-          node-version: 12
-      - run: yarn install
-      - run: yarn build
-      - run: yarn test

.github/workflows/release.yml

@@ -5,7 +5,17 @@ on:
     types: [published]
 
 env:
+  # [avoid issue when Github action upgraded from 18.04 default to 20.04](https://github.com/aws/aws-cli/issues/5262)
+  AWS_EC2_METADATA_DISABLED: true
   REPO_NAME: magda-minion-visualization
+  # Github account username (used for access github registry)
+  GH_USERNAME: magdabot
+  # Github Orgnisation name or user name for this repo
+  GH_ORGNAME: magda-io
+  #Docker Hub username
+  DH_USERNAME: magdabot
+  #S3 bucket name: this s3 bucket will be used to store published helm chart and index
+  S3_BUCKET: magda-charts
 
 jobs:
   release-helm-chart:
@@ -21,50 +31,49 @@ jobs:
       - run: yarn install
       - run: yarn build
       - run: yarn test
-
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.2.0
 
-      - run: yarn helm-lint
+      - name: helm-build-dependencies
+        run: yarn update-all-charts
+
+      - name: helm-check
+        run: yarn helm-lint
+
+      - name: helm-chart-version-check
+        run: yarn check-helm-chart-version deploy/${REPO_NAME}/Chart.yaml
 
       - name: Login to GitHub Package Repository
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
-        run: docker login docker.pkg.github.com -u magdabot -p ${GH_TOKEN}
+          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
+        run: docker login docker.pkg.github.com -u ${GH_USERNAME} -p ${GH_TOKEN}
 
       - name: Build Docker Image & Push to Github Registry
-        run: yarn docker-build-prod --repository=docker.pkg.github.com/magda-io/${REPO_NAME} --name=${REPO_NAME}
+        run: yarn docker-build-prod --repository=docker.pkg.github.com/${GH_ORGNAME}/${REPO_NAME} --name=${REPO_NAME}
 
       - name: Login to Docker Hub
         env:
           DH_TOKEN: ${{ secrets.DOCKER_HUB_PASSWORD }}
-        run: docker login -u magdabot -p ${DH_TOKEN}
+        run: docker login -u ${DH_USERNAME} -p ${DH_TOKEN}
 
       - name: Re-tag & Push Docker Image to Docker Hub
-        run: yarn retag-and-push --fromPrefix=docker.pkg.github.com/magda-io/${REPO_NAME}/ --fromName=${REPO_NAME}
+        run: yarn retag-and-push --fromPrefix=docker.pkg.github.com/${GH_ORGNAME}/${REPO_NAME}/ --fromName=${REPO_NAME}
 
       - name: Configure Git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
-      - name: Setup AWS CLi Tools
-        uses: chrislennon/action-aws-cli@v1.1
-
       - name: Release Helm Chart
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_ACCESS_TOKEN }}"
+          CR_TOKEN: "${{ secrets.GH_ACCESS_TOKEN }}"
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           cd deploy
           mkdir -p sync_dir
           mkdir -p index_dir
-          if ! aws s3 cp s3://magda-charts/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
+          if ! aws s3 cp s3://${S3_BUCKET}/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
           helm package -d sync_dir ${REPO_NAME}
           helm repo index --merge "index_dir/index.yaml" sync_dir
           mv -f sync_dir/index.yaml index_dir/index.yaml
-          aws s3 sync sync_dir s3://magda-charts/ --acl public-read
-          aws s3 cp index_dir/index.yaml s3://magda-charts/index.yaml --acl public-read
+          aws s3 sync sync_dir s3://${S3_BUCKET}/
+          aws s3 cp index_dir/index.yaml s3://${S3_BUCKET}/index.yaml

.gitignore

@@ -50,3 +50,5 @@ magda-tenant-api/lib/
 
 #typescript
 tsconfig.tsbuildinfo
+
+deploy/*/charts

.helmdocsignore

@@ -0,0 +1 @@
+deploy/*/charts

CHANGES.md

@@ -0,0 +1,5 @@
+# 1.0.0
+
+-   Upgrade dependencies
+-   Upgrade CI scripts
+-   Related to https://github.com/magda-io/magda/issues/3229, Use magda-common for docker image related logic

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:10
+FROM node:12-alpine
 
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app

README.md

@@ -29,3 +29,27 @@ Docker image releases can be found from Docker Hub:
 https://hub.docker.com/r/data61/magda-minion-visualization/
 
 Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-minion-visualization/packages) and accessible with access token.
+
+## Requirements
+
+Kubernetes: `>= 1.14.0-0`
+
+| Repository              | Name         | Version       |
+| ----------------------- | ------------ | ------------- |
+| https://charts.magda.io | magda-common | 1.0.0-alpha.4 |
+
+## Values
+
+| Key                          | Type   | Default                                  | Description |
+| ---------------------------- | ------ | ---------------------------------------- | ----------- |
+| defaultAdminUserId           | string | `"00000000-0000-4000-8000-000000000000"` |             |
+| defaultImage.imagePullSecret | bool   | `false`                                  |             |
+| defaultImage.pullPolicy      | string | `"IfNotPresent"`                         |             |
+| defaultImage.repository      | string | `"docker.io/data61"`                     |             |
+| global.image                 | object | `{}`                                     |             |
+| global.minions.image         | object | `{}`                                     |             |
+| global.rollingUpdate         | object | `{}`                                     |             |
+| image.name                   | string | `"magda-minion-visualization"`           |             |
+| resources.limits.cpu         | string | `"100m"`                                 |             |
+| resources.requests.cpu       | string | `"50m"`                                  |             |
+| resources.requests.memory    | string | `"200Mi"`                                |             |

README.md.gotmpl

@@ -0,0 +1,39 @@
+## Magda Visualization Minion
+
+![CI Workflow](https://github.com/magda-io/magda-minion-visualization/workflows/Main%20CI%20Workflow/badge.svg?branch=master) [![Release](https://img.shields.io/github/release/magda-io/magda-minion-visualization.svg)](https://github.com/magda-io/magda-minion-visualization/releases)
+
+A [Magda](https://github.com/magda-io/magda) minion is a service that listens for new records or changes to existing records, performs some kind of operation and then writes the result back to the registry. For instance, we have a broken link minion that listens for changes to distributions, retrieves the URLs described, records whether they were able to be accessed successfully and then writes that back to the registry in its own aspect.
+
+Other aspects exist that are written to by many minions - for instance, we have a "quality" aspect that contains a number of different quality ratings from different sources, which are averaged out and used by search.
+
+This magda minion looks into CSV data files and generates useful information for presenting data with charts (e.g. column data type).
+
+### Helm Chart
+
+It's recommanded to deploy minions with as [dependencies](https://helm.sh/docs/topics/chart_best_practices/dependencies/) of a Magda helm deployment. Example can be found from [here](https://github.com/magda-io/magda-config).
+
+-   Magda Helm Charts Repository Url: https://charts.magda.io
+
+The [helm chart](https://helm.sh/docs/topics/charts/) for this minion is auto released when a [Github Release](https://help.github.com/en/github/administering-a-repository/creating-releases) is created for this repo.
+
+-   Add repository to helm:
+
+```bash
+helm repo add magda-io https://charts.magda.io
+```
+
+### Docker Image
+
+Docker image releases can be found from Docker Hub:
+
+https://hub.docker.com/r/data61/magda-minion-visualization/
+
+Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-minion-visualization/packages) and accessible with access token.
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesHeader" . }}
+
+{{ template "chart.valuesTable" . }}

deploy/magda-minion-visualization/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: magda-common
+  repository: https://charts.magda.io
+  version: 1.0.0-alpha.4
+digest: sha256:a5dcf2df16ca5a3972f92d91434c2e78be0ce411994d2a3cb89c674a711ddc24
+generated: "2021-10-12T15:55:03.187009+11:00"

deploy/magda-minion-visualization/Chart.yaml

@@ -1,8 +1,13 @@
 apiVersion: v2
 name: magda-minion-visualization
 description: A Helm chart for Magda Visualization Minion
-version: "0.0.57-0"
+version: "1.0.0"
 kubeVersion: ">= 1.14.0-0"
 home: "https://github.com/magda-io/magda-minion-visualization"
-sources: ["https://github.com/magda-io/magda-minion-visualization"]
-
+sources: [ "https://github.com/magda-io/magda-minion-visualization" ]
+annotations:
+  magdaModuleType: "minio"
+dependencies:
+  - name: magda-common
+    version: "1.0.0-alpha.4"
+    repository: "https://charts.magda.io"

deploy/magda-minion-visualization/templates/deployment.yaml

@@ -15,18 +15,11 @@ spec:
       labels:
         service: minion-visualization
     spec:
-{{- /*
-See chart value file for details of the logic used to generate this setting value below.!!!
-*/}}
-{{- $imagePullSecret := (ne (.Values.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.image.imagePullSecret ( (ne (.Values.global.minions.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.global.minions.image.imagePullSecret (  (ne (.Values.global.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.global.image.imagePullSecret .Values.defaultImage.imagePullSecret )  ) -}}
-      {{- if ne ($imagePullSecret | toString) "false" }}
-      imagePullSecrets:
-      - name: {{ $imagePullSecret }}
-      {{- end }}
+      {{- include "magda.imagePullSecrets" . | indent 6 }}
       containers:
       - name: minion-visualization
-        image: "{{ .Values.image.repository | default .Values.global.minions.image.repository | default .Values.global.image.repository | default .Values.defaultImage.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Values.global.minions.image.tag | default .Values.global.image.tag | default .Values.defaultImage.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy | default .Values.global.minions.image.pullPolicy | default .Values.global.image.pullPolicy | default .Values.defaultImage.pullPolicy }}
+        image: {{ include "magda.image" . | quote }}
+        imagePullPolicy: {{ include "magda.imagePullPolicy" . | quote }}
 {{- if .Values.global.enableLivenessProbes }}
         livenessProbe:
           httpGet:

deploy/magda-minion-visualization/values.yaml

@@ -7,13 +7,11 @@ global:
 image: 
   name: "magda-minion-visualization"
   # repository: 
-  # tag: 
   # pullPolicy: 
   # imagePullSecret: 
 
 defaultImage:
   repository: docker.io/data61
-  tag: 0.0.57-0
   pullPolicy: IfNotPresent
   imagePullSecret: false
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@magda/minion-visualization",
   "description": "MAGDA visualization Minion",
-  "version": "0.0.57-0",
+  "version": "1.0.0",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "build": "yarn run compile",
@@ -13,25 +13,30 @@
     "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto",
     "retag-and-push": "retag-and-push",
     "test": "echo \"Test cases for visualization minion are yet to be added.\"",
-    "helm-lint": "helm lint deploy/magda-minion-visualization -f deploy/test-deploy.yaml"
+    "helm-lint": "helm lint deploy/magda-minion-visualization -f deploy/test-deploy.yaml",
+    "helm-docs": "helm-docs -t ./README.md.gotmpl -o ../../README.md",
+    "update-all-charts": "helm dep up ./deploy/magda-minion-visualization",
+    "add-all-chart-version-changes": "git ls-files -m | grep Chart.yaml | xargs git add && git ls-files -m | grep Chart.lock | xargs git add",
+    "add-all-helm-docs-changes": "yarn helm-docs && git ls-files -m | grep -i readme.md | xargs git add",
+    "version": "yarn update-helm-chart-version && yarn update-all-charts && yarn add-all-chart-version-changes && yarn add-all-helm-docs-changes"
   },
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@magda/scripts": "^0.0.57-0",
+    "@magda/ci-utils": "^1.0.2",
+    "@magda/docker-utils": "^0.0.60",
     "@types/papaparse": "^4.1.31",
     "@types/read-pkg-up": "^3.0.1",
-    "@types/urijs": "^1.15.31",
     "husky": "^3.1.0",
     "prettier": "^1.19.1",
     "pretty-quick": "^2.0.1",
     "rimraf": "^3.0.0",
-    "typescript": "^3.7.2"
+    "typescript": "^3.9.5"
   },
   "dependencies": {
-    "@magda/minion-sdk": "^0.0.57-0",
-    "@magda/registry-aspects": "^0.0.57-0",
-    "@magda/utils": "^0.0.57-0",
+    "@magda/minion-sdk": "^0.0.60",
+    "@magda/registry-aspects": "^0.0.60",
+    "@magda/utils": "^0.0.60",
     "moment": "^2.19.1",
     "papaparse": "^4.3.6",
     "read-pkg-up": "^3.0.0",