feat: add the ElasticSearch helm chart for cluster-level ES

.gitignore

@@ -6,3 +6,4 @@ infra-*/.terraform*
 infra-*/secrets.auto.tfvars
 my-notes
 values.yaml
+tutor-multi-chart/charts/*.tgz

README.md

@@ -141,7 +141,31 @@ of the box.
 
 ----------------
 
-## Appendix A: how to uninstall this chart
+## Configuration
+
+### Multi-tenant Elasticsearch
+
+Tutor creates an Elasticsearch pod as part of the Kubernetes deployment. Depending on the number of instances
+Memory and CPU use can be lowered by running a central ES cluster instead of an ES pod for every instance.
+
+To enable set `elasticsearch.enabled=true` in your `values.yaml` and deploy the chart.
+
+For each instance you would like to enable this on, set the configuration values in the respective `config.yml`:
+
+```yaml
+K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH: true
+RUN_ELASTICSEARCH: false
+```
+
+- And create the user on the cluster with `tutor k8s harmony create-elasticsearch-user`.
+- Rebuild your Open edX image `tutor images build openedx`.
+- Finally, redeploy your changes: `tutor k8s start && tutor k8s init`.
+
+#### Caveats
+
+In order for SSL to work without warnings the CA certificate needs to be mounted in the relevant pods. This is not yet implemented as due to an [outstanding issue in tutor](https://github.com/overhangio/tutor/issues/791) that had not yet been completed at the time of writing.
+
+## Appendix : how to uninstall this chart
 
 Just run `helm uninstall --namespace tutor-multi tutor-multi` to uninstall this.
 

infra-example/k8s-cluster/main.tf

@@ -26,10 +26,10 @@ resource "digitalocean_kubernetes_cluster" "cluster" {
 
   node_pool {
     name       = "${var.cluster_name}-nodes"
-    size       = "s-2vcpu-4gb"
-    # At this size, at least 4 nodes are recommended to run 2 Open edX instances using the default Tutor images, because
+    size       = "s-4vcpu-8gb"
+    # At this size, at least 3 nodes are recommended to run 2 Open edX instances using the default Tutor images, because
     # resources like MySQL/MongoDB are not shared.
-    min_nodes  = 4
+    min_nodes  = 3
     max_nodes  = 4
     auto_scale = true
   }

tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/commands.py

@@ -0,0 +1,44 @@
+import os
+
+import click
+from tutor import config as tutor_config
+from tutor import env as tutor_env
+from tutor.commands.k8s import K8sContext, kubectl_exec
+from .elasticsearch import ElasticSearchAPI
+
+@click.group(help="Commands and subcommands of the openedx-k8s-harmony.")
+@click.pass_context
+def harmony(context: click.Context) -> None:
+    context.obj = K8sContext(context.obj.root)
+
+
+@click.command(help="Create or update Elasticsearch users")
+@click.pass_obj
+def create_elasticsearch_user(context: click.Context):
+    """
+    Creates or updates the Elasticsearch user
+    """
+    config = tutor_config.load(context.root)
+    namespace = config["K8S_HARMONY_NAMESPACE"]
+    api = ElasticSearchAPI(namespace)
+    username, password = config["ELASTICSEARCH_HTTP_AUTH"].split(":", 1)
+    role_name = f"{username}_role"
+
+    prefix = config["ELASTICSEARCH_INDEX_PREFIX"]
+    api.post(
+        f"_security/role/{role_name}",
+        {"indices": [{"names": [f"{prefix}*"], "privileges": ["all"]}]},
+    )
+
+    api.post(
+        f"_security/user/{username}",
+        {
+            "password": password,
+            "enabled": True,
+            "roles": [role_name],
+            "full_name": username,
+        },
+    )
+
+
+harmony.add_command(create_elasticsearch_user)

tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/elasticsearch.py

@@ -0,0 +1,70 @@
+import json
+import typing
+
+from tutor import utils
+
+
+class ElasticSearchAPI:
+    """
+    Helper class to interact with the ElasticSearch
+    API on the deployed cluster.
+    """
+
+    def __init__(self, namespace):
+        self._command_base = [
+            "kubectl",
+            "exec",
+            "--stdin",
+            "--tty",
+            "--namespace",
+            namespace,
+            "elasticsearch-master-0",
+            "--",
+            "bash",
+            "-c",
+        ]
+        self._curl_base = ["curl", "--insecure", "-u", "elastic:${ELASTIC_PASSWORD}"]
+
+    def run_command(self, curl_options) -> typing.Union[dict, bytes]:
+        """
+        Invokes a curl command on the first Elasticsearch pod.
+
+        If possible returns the parsed json from the Elasticsearch response.
+        Otherwise, the raw bytes from the curl command are returned.
+        """
+        response = utils.check_output(
+            *self._command_base, " ".join(self._curl_base + curl_options)
+        )
+        try:
+            return json.loads(response)
+        except (TypeError, ValueError):
+            return response
+
+    def get(self, endpoint):
+        """
+        Runs a GET request on the Elasticsearch cluster with the specified
+        endpoint.
+
+        If possible returns the parsed json from the Elasticsearch response.
+        Otherwise, the raw bytes from the curl command are returned.
+        """
+        return self.run_command(["-XGET", f"https://localhost:9200/{endpoint}"])
+
+    def post(self, endpoint: str, data: dict) -> typing.Union[dict, bytes]:
+        """
+        Runs a POST request on the Elasticsearch cluster with the specified
+        endpoint.
+
+        If possible returns the parsed json from the Elasticsearch response.
+        Otherwise, the raw bytes from the curl command are returned.
+        """
+        return self.run_command(
+            [
+                "-XPOST",
+                f"https://localhost:9200/{endpoint}",
+                "-d",
+                f"'{json.dumps(data)}'",
+                "-H",
+                '"Content-Type: application/json"',
+            ]
+        )

tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/k8s-services

@@ -2,7 +2,7 @@
 # For multi-instance clusters. Allow one central load balancer on the cluster to
 # handle HTTPS certs and forward traffic to each Open edX instance's Caddy
 # instance.
-{%- set HOSTS = [LMS_HOST, CMS_HOST, PREVIEW_HOST, MFE_HOST] + MULTI_K8S_INGRESS_HOST_LIST %}
+{%- set HOSTS = [LMS_HOST, CMS_HOST, PREVIEW_HOST, MFE_HOST] + K8S_HARMONY_INGRESS_HOST_LIST %}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -12,7 +12,7 @@ metadata:
   annotations:
     cert-manager.io/cluster-issuer: tutor-multi-letsencrypt-global
 spec:
-  ingressClassName: {{ MULTI_K8S_INGRESS_CLASS_NAME }}
+  ingressClassName: {{ K8S_HARMONY_INGRESS_CLASS_NAME }}
   rules:
   {%- for host in HOSTS if host is defined %}
   - host: "{{ host }}"

tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-common-settings

@@ -0,0 +1,10 @@
+{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %}
+ELASTICSEARCH_INDEX_PREFIX = "{{ELASTICSEARCH_INDEX_PREFIX}}"
+ELASTIC_SEARCH_CONFIG = [{
+  "use_ssl": True,
+  "host": "elasticsearch-master.{{K8S_HARMONY_NAMESPACE}}.svc.cluster.local",
+  "verify_certs": False,
+  "port": 9200,
+  "http_auth": "{{ ELASTICSEARCH_HTTP_AUTH }}"
+}]
+{% endif %}

tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/patches/openedx-dockerfile-post-python-requirements

@@ -0,0 +1,7 @@
+{% if K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH %}
+# This is needed otherwise the previously installed edx-search
+# package doesn't get replaced. Once the below branch is merged
+# upstream it will no longer be needed.
+RUN pip uninstall -y edx-search
+RUN pip install --upgrade git+https://github.com/open-craft/edx-search.git@keith/prefixed-index-names
+{% endif %}

tutor-contrib-multi-plugin/tutor_multi_k8s_plugin/plugin.py

@@ -3,6 +3,7 @@
 import pkg_resources
 
 from tutor import hooks
+from . import commands
 
 from .__about__ import __version__
 
@@ -19,7 +20,8 @@
         # when installing additional plugins such as tutor-ecommerce or tutor-minio.
         # The workaround is to manually add a list of hosts to be routed to the caddy
         # instance.
-        "INGRESS_HOST_LIST": [ ]
+        "INGRESS_HOST_LIST": [],
+        "ENABLE_SHARED_ELASTICSEARCH": False,
     },
     "overrides": {
         # Don't use Caddy as a per-instance external web proxy, but do still use it
@@ -28,16 +30,19 @@
         # We are using HTTPS
         "ENABLE_HTTPS": True,
     },
+    "unique": {
+        "ELASTICSEARCH_HTTP_AUTH": "{{K8S_NAMESPACE}}:{{ 24|random_string }}",
+        "ELASTICSEARCH_INDEX_PREFIX": "{{K8S_NAMESPACE}}-{{ 4|random_string|lower }}-",
+    },
 }
 
+# Load all configuration entries
 hooks.Filters.CONFIG_DEFAULTS.add_items(
-    [
-        (f"MULTI_K8S_{key}", value)
-        for key, value in config.get("defaults", {}).items()
-    ]
+    [(f"K8S_HARMONY_{key}", value) for key, value in config["defaults"].items()]
 )
 
-hooks.Filters.CONFIG_OVERRIDES.add_items(list(config.get("overrides", {}).items()))
+hooks.Filters.CONFIG_OVERRIDES.add_items(list(config["overrides"].items()))
+hooks.Filters.CONFIG_UNIQUE.add_items(list(config["unique"].items()))
 
 # Load all patches from the "patches" folder
 for path in glob(
@@ -48,3 +53,5 @@
 ):
     with open(path, encoding="utf-8") as patch_file:
         hooks.Filters.ENV_PATCHES.add_item((os.path.basename(path), patch_file.read()))
+
+hooks.Filters.CLI_COMMANDS.add_item(commands.harmony)

tutor-multi-chart/Chart.lock

@@ -5,5 +5,8 @@ dependencies:
 - name: cert-manager
   repository: https://charts.jetstack.io
   version: v1.11.0
-digest: sha256:7e80a648180d0fe471a9f45d2a47edd5da8894ef544b562911fac2e11d7fd411
-generated: "2023-02-02T14:28:19.400979073-04:00"
+- name: elasticsearch
+  repository: https://helm.elastic.co
+  version: 7.17.3
+digest: sha256:833041ca860a77cc220bdc97f7b6af8ff8b6da0c0a021615dc2858138a29bbbd
+generated: "2023-02-26T12:17:23.507774503+02:00"

tutor-multi-chart/Chart.yaml

@@ -25,3 +25,7 @@ dependencies:
   version: "1.11.0"
   repository: https://charts.jetstack.io
   condition: cert-manager.enabled
+
+- name: elasticsearch
+  version: "7.17.3"
+  repository: https://helm.elastic.co

tutor-multi-chart/templates/elasticsearch/secrets.yaml

@@ -0,0 +1,20 @@
+---
+{{- $ca := genCA "elasticca" 1825 }}
+{{- $cert := genSignedCert "elasticsearch-master.{{ Release.Namespace }}.local" nil (list "elasticsearch-master.{{ Release.Namespace }}.local") 1825 $ca }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elasticsearch-certificates
+type: Opaque
+data:
+  "ca.crt": {{ $ca.Cert | b64enc | toYaml | indent 4}}
+  "tls.key": {{ $cert.Key | b64enc | toYaml | indent 4}}
+  "tls.crt": {{ print $cert.Cert $ca.Cert | b64enc | toYaml | indent 4}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elasticsearch-credentials
+type: Opaque
+data:
+  "password": {{ randAlphaNum 32 | b64enc | quote }}

tutor-multi-chart/values.yaml

@@ -13,3 +13,51 @@ cert-manager:
     # Email address associated with the ACME account. Used to notify about expiring
     # certificates.
   email: ""
+
+# Multi-tenant ElasticSearch
+elasticsearch:
+  enabled: false
+
+  # Operators will need to add/update the following setting in each
+  # of their instances by running the commands:
+  # ```
+  # tutor config save --set K8S_HARMONY_ENABLE_SHARED_ELASTICSEARCH=true --set RUN_ELASTICSEARCH=false
+  # tutor harmony create-elasticsearch-user
+  # ```
+  # RUN_ELASTICSEARCH: false
+  # ELASTICSEARCH_PREFIX_INDEX: "username-"
+  # K8S_HARMONY_USE_SHARED_ELASTICSEARCH: true
+  # ELASTICSEARCH_AUTH: "username:actual_password"
+
+  # We will create the relevant certs, because they need to shared
+  # with pods in other namespaces.
+  createCert: false
+  # Authentication is only available in https
+  protocol: https
+
+  # This secret will contain the http certificates.
+  secretMounts:
+    - name: elasticsearch-certificates
+      secretName: elasticsearch-certificates
+      path: /usr/share/elasticsearch/config/certs
+      defaultMode: 0777
+
+  # The password for the elastic user is stored in this secret
+  extraEnvs:
+    - name: ELASTIC_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: elasticsearch-credentials
+          key: password
+
+  esConfig:
+    "elasticsearch.yml": |
+      xpack.security.enabled: true
+      xpack.security.http.ssl.enabled: true
+      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/tls.key
+      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/tls.crt
+      xpack.security.transport.ssl.enabled: true
+      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/tls.key
+      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/tls.crt
+      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca.crt
+      xpack.security.transport.ssl.verification_mode: certificate

values-example.yaml

@@ -5,3 +5,6 @@ ingress-nginx:
 cert-manager:
   # Set your email address here so auto-generated HTTPS certs will work:
   email: "email@example.com"
+
+elasticsearch:
+  enabled: false