feat: chart index (#1759)

Co-authored-by: CasLubbers <clubbers@akamai.com>

.vscode/settings.json

@@ -9,7 +9,7 @@
     "editor.defaultFormatter": "foxundermoon.shell-format"
   },
   "[javascript]": {
-    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[typescript]": {
     "editor.defaultFormatter": "dbaeumer.vscode-eslint"

chart/chart-index/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

chart/chart-index/Chart.lock

@@ -0,0 +1,51 @@
+dependencies:
+- name: argo-cd
+  repository: https://argoproj.github.io/argo-helm
+  version: 6.7.3
+- name: cert-manager
+  repository: https://charts.jetstack.io
+  version: v1.11.4
+- name: cloudnative-pg
+  repository: https://cloudnative-pg.github.io/charts
+  version: 0.18.0
+- name: external-dns
+  repository: https://charts.bitnami.com/bitnami
+  version: 6.20.4
+- name: external-secrets
+  repository: https://charts.external-secrets.io
+  version: 0.6.1
+- name: gitea
+  repository: https://dl.gitea.io/charts
+  version: 5.0.0
+- name: harbor
+  repository: https://helm.goharbor.io
+  version: 1.10.4
+- name: ingress-nginx
+  repository: https://kubernetes.github.io/ingress-nginx
+  version: 4.6.1
+- name: kube-prometheus-stack
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 46.4.1
+- name: metrics-server
+  repository: https://charts.bitnami.com/bitnami
+  version: 6.8.0
+- name: oauth2-proxy
+  repository: https://charts.bitnami.com/bitnami
+  version: 3.7.4
+- name: prometheus-blackbox-exporter
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 7.10.0
+- name: promtail
+  repository: https://grafana.github.io/helm-charts
+  version: 6.11.2
+- name: sealed-secrets
+  repository: https://bitnami-labs.github.io/sealed-secrets/
+  version: 2.14.1
+- name: tekton-pipeline
+  repository: https://cdfoundation.github.io/tekton-helm-chart/
+  version: 1.0.2
+- name: velero
+  repository: https://vmware-tanzu.github.io/helm-charts/
+  version: 5.4.1
+digest: sha256:9c7e5d75c1f8d3befa942e74b4b0896e36db7f040e100ff642a24c7199cdbdca
+generated: "2024-10-14T11:21:37.891213+02:00"

chart/chart-index/Chart.yaml

@@ -0,0 +1,70 @@
+apiVersion: v2
+name: chart-index
+description: APL chart index
+
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: library
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# # This is the version number of the application being deployed. This version number should be
+# # incremented each time you make changes to the application. Versions are not expected to
+# # follow Semantic Versioning. They should reflect the version the application is using.
+# # It is recommended to use it with quotes.
+# appVersion: "1.16.0"
+
+# The below dependencies are used to download Helm chart archive to the charts directory. Note that charts directory is a symlink.
+dependencies:
+  - name: argo-cd
+    version: 6.7.3
+    repository: https://argoproj.github.io/argo-helm
+  - name: cert-manager
+    version: v1.11.4
+    repository: https://charts.jetstack.io
+  - name: cloudnative-pg
+    version: 0.18.0
+    repository: https://cloudnative-pg.github.io/charts
+  - name: external-dns
+    version: 6.20.4
+    repository: https://charts.bitnami.com/bitnami
+  - name: external-secrets
+    version: 0.6.1
+    repository: https://charts.external-secrets.io
+  - name: gitea
+    version: 5.0.0
+    repository: https://dl.gitea.io/charts
+  - name: harbor
+    version: 1.10.4
+    repository: https://helm.goharbor.io
+  - name: ingress-nginx
+    version: 4.6.1
+    repository: https://kubernetes.github.io/ingress-nginx
+  - name: kube-prometheus-stack
+    version: 46.4.1
+    repository: https://prometheus-community.github.io/helm-charts
+  - name: metrics-server
+    version: 6.8.0
+    repository: https://charts.bitnami.com/bitnami
+  - name: oauth2-proxy
+    version: 3.7.4
+    repository: https://charts.bitnami.com/bitnami
+  - name: prometheus-blackbox-exporter
+    version: 7.10.0
+    repository: https://prometheus-community.github.io/helm-charts
+  - name: promtail
+    version: 6.11.2
+    repository: https://grafana.github.io/helm-charts
+  - name: sealed-secrets
+    version: 2.14.1
+    repository: https://bitnami-labs.github.io/sealed-secrets/
+  - name: tekton-pipeline
+    version: 1.0.2
+    repository: https://cdfoundation.github.io/tekton-helm-chart/
+  - name: velero
+    version: 5.4.1
+    repository: https://vmware-tanzu.github.io/helm-charts/

chart/chart-index/Readme.md

@@ -0,0 +1,22 @@
+The chart-index Helm chart allows to manage most of the APL helm chart dependencies (a.k.a. core apps). 
+The chart-index is so-called library Helm chart and cannot be installed by itself. It only defines dependencies in the `chart/chart-index/Chart.yaml` file. Each dependency follows the following format:
+```
+  - name: <chart name>
+    version: <chart version>
+    repository: <chart url>
+```
+,thus Helm knows the chart registry URL, chart name and version.
+
+In the future, the chart-index is going to be combined with Renovate to discover new versions.
+
+Currently, adding a new version of the core app is performed manually:
+1. In the `chart/chart-index/Chart.yaml` file, change a given version in the `dependencies` list.
+2. Call `npm run charts-update`, so Helm charts archives are downloaded to the `charts/` directory
+3. In charts directory unpack the archive to the corresponding directory
+4. Commit your changes: git commit -m'feat: chart upgrade <app-name>'
+5. Perform smoke tests `npm run validate-templates`
+6. Carefully compare the rendered manifests (your feature branch vs main) by executing `bin/compare.sh`
+
+
+Note 1: some Helm charts do not have an official Helm chart repository. Those helm charts cannot be upgraded via the `chart-index`.
+Note 2: some charts resides in different directory name than the original app name, e.g.: argo-cd app resides in charts/argocd directory

chart/chart-index/charts

@@ -0,0 +1 @@
+../../charts

charts/skeleton/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 appVersion: "1.0"
 description: A Helm chart for Kubernetes that will have it's manifests injected at runtime.
-name: ##CHART
+name: skeleton-##CHART
 version: 0.1.0

docs/development.md

@@ -9,7 +9,7 @@ Effective development starts with an understanding the code structure and the re
 - [Integrating core apps](#Integrating-core-apps)
 - [Working with the team-ns chart](#working-with-the-team-ns-chart)
 - [Testing](#testing)
-- [Otomi CLI](#otomi-cli)
+- [APL CLI](#otomi-cli)
 - [Troubleshooting](#troubleshooting)
 
 # Navigating through code
@@ -20,21 +20,21 @@ Effective development starts with an understanding the code structure and the re
 apl-core
 ├── .values                     # Boilerplate for initializing git repository
 ├── adr                         # Architectural Decision Records [read more](https://adr.github.io/madr/)
-├── bin                         # Otomi CLI entrypoint (deprecated)
-├── binzx                       # Otomi CLI entrypoint
-├── chart                       # Helm chart for installing Otomi
-├── charts                      # All other Helm charts that comprise Otomi
+├── bin                         # APL CLI entrypoint (deprecated)
+├── binzx                       # APL CLI entrypoint
+├── chart                       # Helm chart for installing APL and upgrading APL Helm charts
+├── charts                      # All other Helm charts that comprise APL
 ├── docs                        # Documentation
 ├── helmfile.d/helmfile-*.yaml  # Helmfile specs ordered by name and executed accordingly by otomi commands
 ├── helmfile.d/snippets         # Reusable code snippets
 ├── helmfile.tpl                # Additional Helmfiles that do not have corresponding chart and are not parsed on otomi apply|template command
 ├── k8s                         # Kubernetes manifests that before any other chart
-├── src                         # Otomi CLI source code
+├── src                         # APL CLI source code
 ├── tests                       # Values used for testing purposes
 ├── upgrades.yaml               # Upgrade presync hooks
 ├── values                      # Value templates that serves as input to corresponding Helm charts
 ├── values-changes.yaml         # Definitions for performing data migrations
-├── values-schema.yaml          # JSON schema that defines Otomi interface
+├── values-schema.yaml          # JSON schema that defines APL interface
 └── versions.yaml               # Version tags of otomi-api, otomi-console and otomi-tasks
 ```
 
@@ -56,13 +56,13 @@ Whenever you see `<<: *somename` then it means that [node anchor](<(https://yaml
 
 # Values repo and data flow
 
-A values repo is provided by a user. If Otomi is a function then `values repo` is input arguments. It is composed of many YAML files containing the configuration for various apps and teams.
+A values repo is provided by a user. If APL is a function then `values repo` is input arguments. It is composed of many YAML files containing the configuration for various apps and teams.
 
-While rendering kubernetes manifests Otomi leverages Helmfile.
+While rendering kubernetes manifests APL leverages Helmfile.
 
 > Helmfile is a declarative spec for deploying helm charts. You are encouraged to read more about Helmfile at https://github.com/helmfile/helmfile.
 
-In Otomi, all Helmfile specs are defined in the `helmfile.d/` directory and executed in alphabetical order. The majority of Helmfile specs has the following structure:
+In APL, all Helmfile specs are defined in the `helmfile.d/` directory and executed in alphabetical order. The majority of Helmfile specs has the following structure:
 
 ```go-template
 #helmfiled./999-helmfile.yaml
@@ -106,7 +106,7 @@ flowchart LR
 
 From the flow diagram, we can distinguish four stages of data, before `Kubernetes manifests` are rendered. These are: `Values repo`, `Helmfile bases`, `Helmfile release`, and `Helm chart`.
 
-**Values repo**: It contains files that define input parameters for Otomi. This is where you can define teams, team, services, enabled applications and their configurations, etc. A user sets the `$ENV_DIR` env variable, so Otomi knows about its location.
+**Values repo**: It contains files that define input parameters for APL. This is where you can define teams, team, services, enabled applications and their configurations, etc. A user sets the `$ENV_DIR` env variable, so APL knows about its location.
 
 **Helmfile bases**: From the flow diagram, three files incorporate the content of the `.Values` - a Helmfile variable, which is accessible while using Go templates. These files are merged together in the following order: `snippets/default.yaml` -> `snippets/env.gotmpl` -> `snippets/derived.gotmpl`.
 
@@ -124,7 +124,7 @@ Almost each Helmfile spec loads `snippets/templates.gotmpl` file, which contains
 
 # Validating data from the values repo
 
-Otomi validates all parameters that a user can set in values repo by means checking values against JSON schema defined in the `values-schema.yaml` file. The validation can performed by calling `otomi validate-values` CLI command.
+APL validates all parameters that a user can set in values repo by means checking values against JSON schema defined in the `values-schema.yaml` file. The validation can performed by calling `otomi validate-values` CLI command.
 
 The schema is also a great source of documentation as most of the defined properties have corresponding documentation.
 
@@ -270,7 +270,7 @@ If your app has some parameters that a user should manipulate then make sure you
 
 ## Configuring Namespaces
 
-Otomi defines Kubernetes namespaces and their labels in the `core.yaml` file, at the `k8s.namespaces` property.
+APL defines Kubernetes namespaces and their labels in the `core.yaml` file, at the `k8s.namespaces` property.
 
 ## Configuring Ingress
 
@@ -307,19 +307,19 @@ Every team is deployed as a separate Helmfile release, thus targeting a specific
 otomi template -l name=team-ns-demo
 ```
 
-# Otomi CLI
+# APL CLI
 
 ## Developing CLI
 
 TBD
 
 ## Using CLI while developing templates
 
-Using Otomi CLI can be very helpful while integrating apps or developing new features that involve the execution of Helmfile because it allows you to render and validate manifests. It is possible to use Otomi CLI in development mode, so the Otomi CLI reflects changes made in your local `apl-core` directory.
+Using APL CLI can be very helpful while integrating apps or developing new features that involve the execution of Helmfile because it allows you to render and validate manifests. It is possible to use APL CLI in development mode, so the APL CLI reflects changes made in your local `apl-core` directory.
 
-To run Otomi CLI in the development mode, you must:
+To run APL CLI in the development mode, you must:
 
-- execute Otomi CLI commands from a root directory of the `apl-core` project
+- execute APL CLI commands from a root directory of the `apl-core` project
 - export `ENV_DIR`
 
 First, run `npm install` to build all modules required for CLI.
@@ -344,7 +344,7 @@ export ENV_DIR=$HOME/otomi-values
 otomi bootstrap
 ```
 
-3. Now open `$ENV_DIR` directory in your favorite IDE. Otomi has bootstrapped the skeleton of the repo with default values.
+3. Now open `$ENV_DIR` directory in your favorite IDE. APL has bootstrapped the skeleton of the repo with default values.
 4. Last but not least provide information about your k8s cluster in `$ENV_DIR/env/cluster.yaml` file. Note, it can be fake data if you are not willing to deploy your changes to the cluster.
 
 ```
@@ -360,7 +360,7 @@ cluster:
 otomi validate-values
 ```
 
-Voila. You have built your values repo and can use it for Otomi development.
+Voila. You have built your values repo and can use it for APL development.
 
 Below you can find some useful use cases:
 
@@ -406,15 +406,15 @@ otomi x helmfile -l name=myapp write-values
 
 # Troubleshooting
 
-Some cloud providers are suing custom plugins to refresh the token. Since Otomi CLI executes by default in container some plugins may not be available. In order to solve this issue you can instruct Otomi CLI to execute directly on your host.
+Some cloud providers are suing custom plugins to refresh the token. Since APL CLI executes by default in container some plugins may not be available. In order to solve this issue you can instruct APL CLI to execute directly on your host.
 
 First ensure that you have all required binaries
 
 ```
 npm run install-deps
 ```
 
-Then instruct Otomi to not run in docker:
+Then instruct APL to not run in docker:
 
 ```
 export IN_DOCKER=false

package.json

@@ -118,7 +118,9 @@
   },
   "scripts": {
     "install-deps": "bin/install-deps.sh",
-    "app-versions:csv": "echo 'name,appVersion,chartVersion'; for f in $(find charts -name Chart.yaml -type f -maxdepth 2| sort); do yq e -o=json -I=0 $f | jq -rc '. | [.name, .appVersion, .version] | @csv' | tr -d '\"'; done",
+    "app-versions:csv": "echo 'name,appVersion,chartVersion'; for f in $(find charts -name Chart.yaml -type f -maxdepth 2| sort); do yq eval -o=json $f | jq -rc '. | [.name, .appVersion, .version] | @csv' | tr -d '\"'; done",
+    "charts-update": "cd chart/chart-index && helm dep update",
+    "charts-gen-deps": "for f in $(find charts -name Chart.yaml -type f -maxdepth 2| sort); do yq eval $f -o=json | jq '{name, version, repository}';done |   jq -s '.' | yq eval -P",
     "adr": "adr-log -d adr -i",
     "check-policies": "ENV_DIR=$PWD/tests/fixtures NODE_ENV=test binzx/otomi check-policies",
     "clean": "rm -rf dist >/dev/null",