Merge pull request #399 from cliveseldon/ksonnet_analytics

Provide Ksonnet Analytics Package

docs/install.md

@@ -87,4 +87,7 @@ Notes
 
 # Next Steps
 
- * [Jupyter notebooks showing worked examples](../readme.md#quick-start)
+ * [Jupyter notebooks showing worked examples](../readme.md#quick-start)
+ * Seldon Core Analytics (example Prometheus and Grafana)
+   * [Helm Chart](../helm-charts/seldon-core-analytics)
+   * [Ksonnet Package](../seldon-core/seldon-core-analytics)

helm-charts/seldon-core-analytics/templates/alertmanager-deployment.json

@@ -13,7 +13,7 @@
                 "template": {
                     "metadata": {
                         "annotations": {
-                            "checksum/config-1": {{ include (print $.Template.BasePath "/alertmanager-config-configmap.yaml") . | sha256sum | quote }},
+                            "checksum/config-1": {{ include (print $.Template.BasePath "/alertmanager-config-configmap.yaml") . | sha256sum | quote }}
                         },
                         "labels": {
                             "app": "alertmanager-server"

helm-charts/seldon-core-analytics/values.yaml

@@ -4,7 +4,7 @@ alertmanager:
 grafana_prom_service_type: NodePort
 grafana_prom_admin_password: admin
 persistence:
-  enabled: true
+  enabled: false
 rbac:
   enabled: true
 prometheus:

seldon-core/seldon-core-analytics/README.md

@@ -0,0 +1,23 @@
+# Seldon Core Analytics
+
+This package provides an example Prometheus and Grafana install with default dashboard for Seldon Core. Its only intended for demonstration. In production you should create your own Prometheus and Grafana installation.
+
+## Install
+
+ * Install seldon-core as described [here](../../docs/install.md#with-ksonnet)
+ * Install seldon-core-analytics. Parameters:
+   * ```password``` set the Grafana dashboard password - default 'admin'
+   * ```prometheusServiceType``` the Prometheus service type - default 'ClusterIP'
+   * ```grafanaServiceType``` the Grafama service type - default `NodePort`
+
+```
+    ks pkg install seldon-core/seldon-core-analytics@master
+    ks generate seldon-core-analytics seldon-core-analytics
+```
+ * Launch components onto cluster
+ ```
+ ks apply default
+ ```
+Notes
+
+ * You can use ```--namespace``` to install seldon-core to a particular namespace

seldon-core/seldon-core-analytics/analytics.libsonnet

@@ -0,0 +1,153 @@
+local k = import "k.libsonnet";
+local deployment = k.extensions.v1beta1.deployment;
+local container = k.apps.v1beta1.deployment.mixin.spec.template.spec.containersType;
+local service = k.core.v1.service.mixin;
+local serviceAccountMixin = k.core.v1.serviceAccount.mixin;
+local clusterRoleBindingMixin = k.rbac.v1beta1.clusterRoleBinding.mixin;
+local clusterRoleBinding = k.rbac.v1beta1.clusterRoleBinding;
+local roleBindingMixin = k.rbac.v1beta1.roleBinding.mixin;
+local roleBinding = k.rbac.v1beta1.roleBinding;
+local roleMixin = k.rbac.v1beta1.role.mixin;
+local serviceAccount = k.core.v1.serviceAccount;
+local secret = k.core.v1.secret;
+local configMap = k.core.v1.configMap;
+local job = k.batch.v1.job;
+local daemonSet = k.extensions.v1beta1.daemonSet;
+
+local analyticsTemplate = import "json/analytics.json";
+
+local getGrafanaPromSecret(x) = std.endsWith(x.metadata.name, "grafana-prom-secret");
+local getAlertmanagerServerConf(x) = std.endsWith(x.metadata.name, "alertmanager-server-conf") && x.kind == "ConfigMap";
+local getGrafanaImportDashboards(x) = std.endsWith(x.metadata.name, "grafana-import-dashboards") && x.kind == "ConfigMap";
+local getPrometheusRules(x) = std.endsWith(x.metadata.name, "prometheus-rules") && x.kind == "ConfigMap";
+local getPrometheusServerConf(x) = std.endsWith(x.metadata.name, "prometheus-server-conf") && x.kind == "ConfigMap";
+local getPrometheusClusterRole(x) = x.metadata.name == "prometheus" && x.kind == "ClusterRole";
+local getPrometheusServiceAccount(x) = x.metadata.name == "prometheus" && x.kind == "ServiceAccount";
+local getPrometheusClusterRoleBinding(x) = x.metadata.name == "prometheus" && x.kind == "ClusterRoleBinding";
+local getAlertManagerDeployment(x) = x.metadata.name == "alertmanager-deployment" && x.kind == "Deployment";
+local getAlertManagerService(x) = x.metadata.name == "alertmanager" && x.kind == "Service";
+local getGrafanaPromDeployment(x) = x.metadata.name == "grafana-prom-deployment" && x.kind == "Deployment";
+local getGrafanaPromService(x) = x.metadata.name == "grafana-prom" && x.kind == "Service";
+local getGrafanaPromJob(x) = x.metadata.name == "grafana-prom-import-dashboards" && x.kind == "Job";
+local getPrometheusExporter(x) = x.metadata.name == "prometheus-node-exporter" && x.kind == "DaemonSet";
+local getPrometheusExporterService(x) = x.metadata.name == "prometheus-node-exporter" && x.kind == "Service";
+local getPrometheusDeployment(x) = x.metadata.name == "prometheus-deployment" && x.kind == "Deployment";
+local getPrometheusService(x) = x.metadata.name == "prometheus-seldon" && x.kind == "Service";
+
+{
+  parts(name, namespace)::
+
+    {
+      grafanaPromSecret(password)::
+
+        local baseGrafanaPromSecret = std.filter(getGrafanaPromSecret, analyticsTemplate.items)[0];
+	baseGrafanaPromSecret +
+		secret.withDataMixin({"grafana-prom-admin-password": std.base64(password)}) +
+		secret.mixin.metadata.withNamespace(namespace),
+
+      alertManagerServerConf()::
+
+        local baseAlertManagerServerConf = std.filter(getAlertmanagerServerConf, analyticsTemplate.items)[0];
+	baseAlertManagerServerConf +
+		configMap.mixin.metadata.withNamespace(namespace),
+
+      grafanaImportDashboards()::
+
+        local baseGrafanaImportDashboards = std.filter(getGrafanaImportDashboards, analyticsTemplate.items)[0];
+	baseGrafanaImportDashboards +
+		configMap.mixin.metadata.withNamespace(namespace),
+
+      prometheusRules()::
+
+        local basePrometheusRules = std.filter(getPrometheusRules, analyticsTemplate.items)[0];
+	basePrometheusRules +
+		configMap.mixin.metadata.withNamespace(namespace),	
+
+      prometheusServerConf()::
+
+        local basePrometheusServerConf = std.filter(getPrometheusServerConf, analyticsTemplate.items)[0];
+	basePrometheusServerConf +
+		configMap.mixin.metadata.withNamespace(namespace),		
+
+      prometheusClusterRole()::
+
+        std.filter(getPrometheusClusterRole, analyticsTemplate.items)[0],
+
+      prometheusServiceAccount()::
+
+        local basePrometheusServiceAccount = std.filter(getPrometheusServiceAccount, analyticsTemplate.items)[0];
+	basePrometheusServiceAccount +
+		serviceAccountMixin.metadata.withNamespace(namespace),
+
+      prometheusClusterRoleBinding()::
+
+        local rbacClusterRoleBinding = std.filter(getPrometheusClusterRoleBinding, analyticsTemplate.items)[0];
+
+        local subject = rbacClusterRoleBinding.subjects[0]
+                        { namespace: namespace };
+
+        rbacClusterRoleBinding +
+        clusterRoleBindingMixin.metadata.withNamespace(namespace) +
+        clusterRoleBinding.withSubjects([subject]),
+
+
+      alertManagerDeployment()::
+
+        local baseAlertManagerDeployment = std.filter(getAlertManagerDeployment, analyticsTemplate.items)[0];
+	baseAlertManagerDeployment +
+		deployment.mixin.metadata.withNamespace(namespace),
+
+      alertManagerService()::
+
+        local baseAlertManagerService = std.filter(getAlertManagerService, analyticsTemplate.items)[0];
+	baseAlertManagerService +
+		service.metadata.withNamespace(namespace),
+
+      grafanaPromDeployment()::
+
+        local baseGrafanaDeployment = std.filter(getGrafanaPromDeployment, analyticsTemplate.items)[0];
+	baseGrafanaDeployment +
+		deployment.mixin.metadata.withNamespace(namespace),
+
+      grafanaPromService(serviceType)::
+
+        local baseServiceGrafana = std.filter(getGrafanaPromService, analyticsTemplate.items)[0];
+
+	baseServiceGrafana +
+		service.spec.withType(serviceType) +
+		service.metadata.withNamespace(namespace),
+
+      grafanaPromJob()::
+
+        local baseGrafanaPromJob = std.filter(getGrafanaPromJob, analyticsTemplate.items)[0];
+	baseGrafanaPromJob +
+		job.mixin.metadata.withNamespace(namespace),
+
+      prometheusExporter()::
+
+        local basePrometheuExporter = std.filter(getPrometheusExporter, analyticsTemplate.items)[0];
+	basePrometheuExporter +
+		daemonSet.mixin.metadata.withNamespace(namespace),
+
+      prometheusExporterService()::
+
+        local basePrometheusExporterService = std.filter(getPrometheusExporterService, analyticsTemplate.items)[0];
+	basePrometheusExporterService +
+		service.metadata.withNamespace(namespace),
+
+      prometheusDeployment()::
+
+        local basePrometheusDeployment = std.filter(getPrometheusDeployment, analyticsTemplate.items)[0];
+	basePrometheusDeployment +
+		deployment.mixin.metadata.withNamespace(namespace),	
+
+      prometheusService(serviceType)::
+
+        local baseServiceProm = std.filter(getPrometheusService, analyticsTemplate.items)[0];
+
+	baseServiceProm +
+	 	service.spec.withType(serviceType) +
+		service.metadata.withNamespace(namespace),
+
+    },  // parts
+}

seldon-core/seldon-core-analytics/json/README.md

@@ -0,0 +1 @@
+analytics.json is created from ../../../util/ksonnet scripts

seldon-core/seldon-core-analytics/parts.yaml

@@ -0,0 +1,37 @@
+{
+   "name": "seldon-core-analytics",
+   "apiVersion": "0.0.1",
+   "kind": "ksonnet.io/parts",
+   "description": "Seldon Core Analytics\n",
+   "author": "seldon-core team <devext@seldon.io>",
+   "contributors": [
+      {
+         "name": "Clive Cox",
+         "email": "cc@seldon.io"
+      }
+   ],
+   "repository": {
+      "type": "git",
+      "url": "https://github.com/SeldonIO/seldon-core/ksonnet/registry"
+   },
+   "bugs": {
+      "url": "https://github.com/SeldonIO/seldon-core/issues"
+   },
+   "keywords": [
+      "kubernetes",
+      "machine learning",
+      "deployment",
+      "grafana",
+      "prometheus"
+   ],
+   "quickStart": {
+      "prototype": "io.ksonnet.pkg.seldon-core",
+      "componentName": "seldon-core-analytics",
+      "flags": {
+         "name": "seldon-core",
+         "namespace": "default"
+      },
+      "comment": "Seldon Core Analytics"
+   },
+   "license": "Apache 2.0"
+}

seldon-core/seldon-core-analytics/prototypes/analytics.jsonnet

@@ -0,0 +1,47 @@
+// @apiVersion 0.1
+// @name io.ksonnet.pkg.seldon-core-analytics
+// @description Seldon Core example Grafana and Prometheus installation
+// @shortDescription Seldon Core Grafana amd Prometheus with example dashboards.
+// @param name string seldon Name to give seldon analytics
+// @optionalParam namespace string null Namespace to use for the components. It is automatically inherited from the environment if not set.
+// @optionalParam password string admin Admin password for Grafana
+// @optionalParam prometheusServiceType string ClusterIP Prometheus service type
+// @optionalParam grafanaServiceType string NodePort Grafana service type
+
+local k = import "k.libsonnet";
+local analytics = import "seldon-core/seldon-core-analytics/analytics.libsonnet";
+
+// updatedParams uses the environment namespace if
+// the namespace parameter is not explicitly set
+local updatedParams = params {
+  namespace: if params.namespace == "null" then env.namespace else params.namespace,
+};
+
+local name = import "param://name";
+local namespace = updatedParams.namespace;
+local password = import "param://password";
+local prometheusServiceType = import "param://prometheusServiceType";
+local grafanaServiceType = import "param://grafanaServiceType";
+
+// Analytics Resources
+local resources = [
+  analytics.parts(name, namespace).grafanaPromSecret(password),
+  analytics.parts(name, namespace).alertManagerServerConf(),
+  analytics.parts(name, namespace).grafanaImportDashboards(),
+  analytics.parts(name, namespace).prometheusRules(),
+  analytics.parts(name, namespace).prometheusServerConf(),
+  analytics.parts(name, namespace).prometheusClusterRole(),
+  analytics.parts(name, namespace).prometheusServiceAccount(),
+  analytics.parts(name, namespace).prometheusClusterRoleBinding(),
+  analytics.parts(name, namespace).alertManagerDeployment(),
+  analytics.parts(name, namespace).alertManagerService(),
+  analytics.parts(name, namespace).grafanaPromDeployment(),
+  analytics.parts(name, namespace).grafanaPromService(grafanaServiceType),
+  analytics.parts(name, namespace).grafanaPromJob(),
+  analytics.parts(name, namespace).prometheusExporter(),
+  analytics.parts(name, namespace).prometheusExporterService(),
+  analytics.parts(name, namespace).prometheusDeployment(),
+  analytics.parts(name, namespace).prometheusService(prometheusServiceType),    
+];
+
+resources

util/ksonnet/Makefile

@@ -1,4 +1,5 @@
-KSONNET_FOLDER=../../seldon-core/seldon-core/json
+SC_KSONNET_FOLDER=../../seldon-core/seldon-core/json
+SCA_KSONNET_FOLDER=../../seldon-core/seldon-core-analytics/json
 
 .PHONY: start-minikube
 start-minikube:
@@ -29,18 +30,34 @@ template_0.1.yaml: seldon-core-release-0.1
 template_0.1.json: template_0.1.yaml kubectl
 	./kubectl convert -f template_0.1.yaml -o json > template_0.1.json
 
+
+template_analytics.txt:
+	helm template ../../helm-charts/seldon-core-analytics > template_analytics.txt
+
+analytics.json: template_analytics.txt
+	python yj2j.py template_analytics.txt > analytics.json
+
+######
+
 .PHONY: build
 build: template_0.2_single_namespace.json template_0.2_cluster_wide.json template_0.1.json
 
+.PHONY: release_analytics
+release_analytics:
+	cp analytics.json ${SCA_KSONNET_FOLDER}
+
 .PHONY: release
 release:
-	cp template_0.2_single_namespace.json ${KSONNET_FOLDER}
-	cp template_0.2_cluster_wide.json ${KSONNET_FOLDER}
-	cp template_0.1.json ${KSONNET_FOLDER}
+	cp template_0.2_single_namespace.json ${SC_KSONNET_FOLDER}
+	cp template_0.2_cluster_wide.json ${SC_KSONNET_FOLDER}
+	cp template_0.1.json ${SC_KSONNET_FOLDER}
 
 .PHONY: clean
 clean:
 	rm -f *.json
 	rm -f *.yaml
 	rm -rf seldon-core-release-0.1
 	rm -f kubectl
+	rm -f template_analytics.txt
+	rm -f analytics.json
+

util/ksonnet/yj2j.py

@@ -0,0 +1,38 @@
+import sys, yaml, json
+import argparse
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input", type=str,help="Input file which contains list of json and yaml from helm template")
+
+    args = parser.parse_args()
+
+    with open(args.input, 'r') as myfile:
+        data=myfile.read()
+
+    parts = data.split("---\n# Source")
+
+    resources = []
+
+    for part in parts[1:]:
+        name = part.split("\n")[0]
+        lines = part.split("\n")[1:]
+        jory = "\n".join(lines)
+        if name.endswith(".yaml"):
+            items = yaml.load_all(jory)
+            for item in items:
+                resources.append(item)
+        else:
+            j = json.loads(jory)
+            if "kind" in j  and j["kind"] == "List":
+                for item in j["items"]:
+                    resources.append(item)
+            else:
+                resources.append(j)
+
+    jlist = {"kind": "List","apiVersion": "v1","metadata": {},"items": resources}
+    json.dump(jlist, sys.stdout, indent=4)
+
+if __name__ == "__main__":
+    main()