fix: performance issues yq #261 (#288)

linode · Feb 4, 2021 · a98e1c8 · a98e1c8
1 parent 08718d7
commit a98e1c8
Show file tree

Hide file tree

Showing 10 changed files with 46 additions and 43 deletions.
diff --git a/bin/bootstrap.sh b/bin/bootstrap.sh
@@ -16,10 +16,11 @@ has_otomi='false'
 function generate_loose_schema() {
   local targetPath="$ENV_DIR/.vscode/values-schema.yaml"
   local sourcePath="$PWD/values-schema.yaml"
-  yq d $sourcePath '**.required.' | yq d - 'properties.toolsVersion' | yq d - 'properties.cluster' >$targetPath
+  yq r -j "${sourcePath}" | jq "del(.. | .required?)" | yq r --prettyPrint - > "${targetPath}"
+  # yq d $sourcePath '**.required.' | yq d - 'properties.toolsVersion' | yq d - 'properties.cluster' >$targetPath
   # also put a copy in the .values folder for local hinting of .demo/env/*.yaml files:
   [ "$PWD" != "/home/app/stack" ] && cp $targetPath .values/
-  echo "Stored JSON schema at: $targetPath"
+  echo "Stored YAML schema at: $targetPath"
 }
 
 # install CLI

diff --git a/bin/otomi b/bin/otomi
@@ -246,12 +246,13 @@ function execute() {
     hf)
       validate_cluster_env
       set -o pipefail
-      drun helmfile -e $CLOUD-$CLUSTER $@ | grep -Ev $helmfile_output_hide
+      drun helmfile -e $CLOUD-$CLUSTER "$@" | grep -Ev $helmfile_output_hide
+
       ;;
     apply)
       validate_cluster_env
       set -o pipefail
-      drun helmfile -e $CLOUD-$CLUSTER $@ apply --skip-deps | grep -Ev $helmfile_output_hide
+      drun helmfile -e $CLOUD-$CLUSTER "$@" apply --skip-deps | grep -Ev $helmfile_output_hide
       ;;
     bash)
       check_kube_context=0
@@ -260,22 +261,22 @@ function execute() {
       ;;
     bootstrap)
       check_kube_context=0
-      if [[ "$@" != '' ]]; then
+      if [[ "$*" != '' ]]; then
         validate_cluster_env
       fi
       drun bin/bootstrap.sh 1
       ;;
     console)
       check_kube_context=0
       evaluate_secrets
-      run_console $@
+      run_console "$@"
       ;;
     decrypt)
       check_sops_file
       check_kube_context=0
       evaluate_secrets
-      if [[ "$@" != '' ]]; then
-        for f in $@; do
+      if [[ "$*" != '' ]]; then
+        for f in "$@"; do
           echo "Decrypting $f"
           drun helm secrets dec ./env/$f >/dev/null
         done
@@ -291,19 +292,19 @@ function execute() {
     diff)
       validate_cluster_env
       set -o pipefail
-      drun helmfile -e $CLOUD-$CLUSTER $@ diff --skip-deps | grep -Ev $helmfile_output_hide
+      drun helmfile -e $CLOUD-$CLUSTER "$@" diff --skip-deps | grep -Ev $helmfile_output_hide
       ;;
     destroy)
       validate_cluster_env
       set -o pipefail
-      drun helmfile -e $CLOUD-$CLUSTER $@ destroy | grep -Ev $helmfile_output_hide
+      drun helmfile -e $CLOUD-$CLUSTER "$@" destroy | grep -Ev $helmfile_output_hide
       ;;
     encrypt)
       check_sops_file
       check_kube_context=0
       evaluate_secrets
-      if [[ "$@" != '' ]]; then
-        for f in $@; do
+      if [[ "$*" != '' ]]; then
+        for f in "$@"; do
           echo "Encrypting $f"
           drun helm secrets enc ./env/$f >/dev/null
         done
@@ -315,13 +316,13 @@ function execute() {
     sync)
       validate_cluster_env
       set -o pipefail
-      drun helmfile -e $CLOUD-$CLUSTER $@ sync --skip-deps | grep -Ev $helmfile_output_hide
+      drun helmfile -e $CLOUD-$CLUSTER "$@" sync --skip-deps | grep -Ev $helmfile_output_hide
       ;;
     template)
       check_kube_context=0
       validate_cluster_env
       set -o pipefail
-      drun helmfile -e $CLOUD-$CLUSTER --quiet $@ template --skip-deps | grep -Ev $helmfile_output_hide_tpl
+      drun helmfile -e $CLOUD-$CLUSTER --quiet "$@" template --skip-deps | grep -Ev $helmfile_output_hide_tpl
       ;;
     rotate-keys)
       check_sops_file
@@ -339,22 +340,17 @@ function execute() {
       evaluate_secrets
       drun bin/gen-drone.sh
       ;;
-    test)
-      validate_cluster_env
-      evaluate_secrets
-      drun bin/test.sh
-      ;;
     validate-templates)
-      if [[ "$@" != '' ]]; then
+      if [[ "$*" != '' ]]; then
         validate_cluster_env
       fi
       check_kube_context=0
       evaluate_secrets
-      drun bin/validate-templates.sh $@
+      drun bin/validate-templates.sh "$@"
       ;;
     x)
       check_kube_context=0
-      drun $@
+      drun "$@"
       ;;
     commit)
       check_kube_context=0
@@ -379,7 +375,7 @@ function execute() {
       ;;
     *)
       show_usage
-      [ -z "$1" ] && err "Unknown command: $@" && exit 1
+      [ -z "$1" ] && err "Unknown command: $*" && exit 1
       ;;
   esac
 }
@@ -388,4 +384,4 @@ function execute() {
 
 set_env_and_stack_dir
 set_helm_config
-execute $command $@
+execute $command "$@"
diff --git a/bin/tests/bootstrap.bats b/bin/tests/bootstrap.bats
@@ -55,4 +55,14 @@ teardown () {
     bin/bootstrap.sh 
     run bin/bootstrap.sh
     assert_success
-}
+}
+
+@test "executing bootstrap.sh creates a valid loose schema" {
+    git init "$ENV_DIR"
+    run bin/bootstrap.sh
+    assert_success
+    assert_file_exist "$ENV_DIR/.vscode/values-schema.yaml"
+
+    result=$(yq r "$ENV_DIR/.vscode/values-schema.yaml" '**.required.' | wc -l)
+    [ "$result" -eq 0 ]
+}
diff --git a/charts/team-ns/values.yaml b/charts/team-ns/values.yaml
@@ -1,4 +1,3 @@
-toolsVersion: 1.3.5
 cluster:
   provider: ""
   name: ""

diff --git a/core.yaml b/core.yaml
@@ -1,4 +1,4 @@
-toolsVersion: 1.4.9
+# NOTE: This file is merged with other values in ./helmfile.d/snippets/env.gotmpl
 
 k8s:
   namespaces:

diff --git a/docs/BATS.md b/docs/BATS.md
@@ -4,11 +4,11 @@ This document is relevant starting from [v0.11.52](https://github.com/redkubes/o
 
 ## Where can I find BATS?
 
-The otomi/tools:1.4.10 includes the [bats framework](https://github.com/bats-core/bats-core), including libraries [bats-assert](https://github.com/ztombol/bats-assert), [bats-file](https://github.com/ztombol/bats-support) and [bats-support](https://github.com/ztombol/bats-support). These links include relevant documentation, such as syntax, which won't be discussed on this page.
+The otomi/tools:1.4.10 or newer includes the [bats framework](https://github.com/bats-core/bats-core), including libraries [bats-assert](https://github.com/ztombol/bats-assert), [bats-file](https://github.com/ztombol/bats-support) and [bats-support](https://github.com/ztombol/bats-support). These links include relevant documentation, such as syntax, which won't be discussed on this page.
 
 Example of calling the binary:
 
-`docker run --rm otomi/tools:1.4.10 bats bin/tests`
+`docker run --rm otomi/core:latest bats bin/tests`
 
 This example assumes tests exist in the container directory `otomi-core/bin/tests`. You can call bats with a directory as parameter and it will execute any `*.bats` file.
 

diff --git a/values-schema.yaml b/values-schema.yaml
@@ -1370,8 +1370,6 @@ properties:
         patternProperties:
           '^[a-z-]+$':
             $ref: '#/definitions/team'
-  toolsVersion:
-    type: string
 required:
   - alerts
   - clouds
@@ -1381,4 +1379,3 @@ required:
   - teamConfig
   - k8s
   - services
-  - toolsVersion
diff --git a/values/jobs/harbor.gotmpl b/values/jobs/harbor.gotmpl
@@ -22,15 +22,15 @@ tasks:
     description: Configure OIDC as a primary auhentication method and populate teams to harbor projects
     init:
       image:
-        repository: {{ $c | get "jobs.harbor.init.image.repository" "otomi/tools" }}
-        tag: {{ $c | get "jobs.harbor.init.image.tag" $v.toolsVersion }}
-        pullPolicy: {{ $c | get "jobs.harbor.init.image.pullPolicy" "IfNotPresent" }}
+        repository: otomi/tools
+        tag: v1.4.11
+        pullPolicy: IfNotPresent
       # move secret for harbor to use
       script: kubectl -n {{ $ns }} get secret harbor-{{ $v.cluster.domain | replace "." "-" }} -o yaml --export | kubectl -n harbor apply -f -
     image:
-      repository: {{ $c | get "jobs.harbor.image.repository" "otomi/tasks" }}
-      tag: {{ $c | get "jobs.harbor.image.tag" "v0.2.1" }}
-      pullPolicy: {{ $c | get "jobs.harbor.image.pullPolicy" "IfNotPresent" }}
+      repository: otomi/tasks
+      tag: v0.2.1
+      pullPolicy: IfNotPresent
     secret:
       HARBOR_PASSWORD:  {{ $h | get "adminPassword" "bladibla" }}
       HARBOR_USER: admin

diff --git a/values/jobs/ingress-azure.gotmpl b/values/jobs/ingress-azure.gotmpl
@@ -6,9 +6,9 @@ tasks:
     type: cronjob
     enabled: false
     image:
-      repository: {{ $c | get "jobs.ingress-azure.image.repository" "otomi/tools" }}
-      tag: {{ $c | get "jobs.ingress-azure.image.tag" $v.toolsVersion }}
-      pullPolicy: {{ $c | get "jobs.ingress-azure.image.pullPolicy" "IfNotPresent" }}
+      repository: otomi/tools
+      tag: v1.4.11
+      pullPolicy: IfNotPresent
     env:
       LABELS: "app=ingress-azure"
       NS: ingress

diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl
@@ -51,8 +51,8 @@ tools:
   enabled: {{ $v.sops.enabled }}
   {{- if $v.sops.enabled }}
   image:
-    pullPolicy: {{ $o | get "tools.image.pullPolicy" "IfNotPresent" }}
-    tag: {{ $o | get "tools.image.tag" "latest" }}
+    pullPolicy: IfNotPresent
+    tag: v1.4.11
     {{- with $o | get "resources.tools" nil }}
     resources:
       {{- toYaml . | nindent 4 }}