feat: add prometheus jmx (#261)

* feat: add prometheus jmx JanssenProject/jans#682 * fix: typo
GluuFederation · Jun 21, 2022 · 867a031 · 867a031
1 parent c767397
commit 867a031
Show file tree

Hide file tree

Showing 12 changed files with 77 additions and 4 deletions.
diff --git a/docker-casa/Dockerfile b/docker-casa/Dockerfile
@@ -59,6 +59,17 @@ RUN python3 -m ensurepip \
     && pip3 install --no-cache-dir -r /app/requirements.txt \
     && pip3 uninstall -y pip wheel
 
+# ==========
+# Prometheus
+# ==========
+
+ARG PROMETHEUS_JAVAAGENT_VERSION=0.17.0
+COPY conf/prometheus-config.yaml /opt/prometheus/
+RUN mkdir -p /opt/prometheus \
+    && wget -q https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${PROMETHEUS_JAVAAGENT_VERSION}/jmx_prometheus_javaagent-${PROMETHEUS_JAVAAGENT_VERSION}.jar -O /opt/prometheus/jmx_prometheus_javaagent.jar \
+    && java -jar ${JETTY_HOME}/start.jar jetty.home=${JETTY_HOME} jetty.base=${JETTY_BASE}/casa --add-module=jmx,stats
+
+
 # =====================
 # jans-linux-setup sync
 # =====================
@@ -200,7 +211,8 @@ ENV CN_MAX_RAM_PERCENTAGE=75.0 \
     CN_SSL_CERT_FROM_SECRETS=false \
     GOOGLE_PROJECT_ID="" \
     GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \
-    GLUU_CASA_ADMIN_LOCK_FILE=/opt/jans/jetty/casa/resources/.administrable
+    GLUU_CASA_ADMIN_LOCK_FILE=/opt/jans/jetty/casa/resources/.administrable \
+    CN_PROMETHEUS_PORT=""
 
 # ==========
 # misc stuff

diff --git a/docker-casa/README.md b/docker-casa/README.md
@@ -72,6 +72,7 @@ The following environment variables are supported by the container:
 - `GOOGLE_PROJECT_ID`: Google Project ID (default to empty string).
 - `GLUU_CASA_APP_LOGGERS`: Custom logging configuration in JSON-string format with hash type (see [Configure app loggers](#configure-app-loggers) section for details).
 - `GLUU_CASA_ADMIN_LOCK_FILE`: Path to lock file to enable/disable administration feature (default to `/opt/jans/jetty/casa/resources/.administrable`). If file is not exist, the feature is disabled.
+- `CN_PROMETHEUS_PORT`: Port used by Prometheus JMX agent (default to empty string). To enable Prometheus JMX agent, set the value to a number. See [Exposing metrics](#exposing-metrics) for details.
 
 ### Configure app loggers
 
@@ -103,6 +104,16 @@ The following key-value pairs are the defaults:
 }
 ```
 
+### Exposing metrics
+
+As per v1.0.1, certain metrics can be exposed via Prometheus JMX exporter.
+To expose the metrics, set the `CN_PROMETHEUS_PORT` environment variable, i.e. `CN_PROMETHEUS_PORT=9093`.
+Afterwards, metrics can be scraped by Prometheus or accessed manually by making request to `/metrics` URL,
+i.e. `http://container:9093/metrics`.
+
+Note that Prometheus JMX exporter uses pre-defined config file (see `conf/prometheus-config.yaml`).
+To customize the config, mount custom config file to `/opt/prometheus/prometheus-config.yaml` inside the container.
+
 ### Hybrid mapping
 
 Hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below:

diff --git a/docker-casa/conf/prometheus-config.yaml b/docker-casa/conf/prometheus-config.yaml
@@ -0,0 +1,10 @@
+---
+startDelaySeconds: 0
+ssl: false
+lowercaseOutputName: true
+lowercaseOutputLabelNames: true
+whitelistObjectNames: ["org.eclipse.jetty.server.handler:*"]
+rules:
+  - pattern: ".*xx"
+  - pattern: ".*requests"
+  - pattern: ".*requestTimeTotal"
diff --git a/docker-casa/scripts/entrypoint.sh b/docker-casa/scripts/entrypoint.sh
@@ -1,6 +1,10 @@
 #!/bin/sh
 set -e
 
+# =========
+# FUNCTIONS
+# =========
+
 get_casa_plugins(){
     # ===============
     # PREPARE PLUGINS
@@ -16,9 +20,16 @@ get_casa_plugins(){
     #&& wget -q ${CASA_PLUGIN_REPO}/duo-plugin/${GLUU_VERSION}/duo-plugin-${GLUU_VERSION}.jar -O ${CASA_PLUGIN_DIRECTORY}/duo-plugin-${GLUU_VERSION}.jar
 }
 
-# =========
-# FUNCTIONS
-# =========
+get_prometheus_opt() {
+    prom_opt=""
+
+    if [ -n "${CN_PROMETHEUS_PORT}" ]; then
+        prom_opt="
+            -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=${CN_PROMETHEUS_PORT}:/opt/prometheus/prometheus-config.yaml
+        "
+    fi
+    echo "${prom_opt}"
+}
 
 # ==========
 # ENTRYPOINT
@@ -45,5 +56,6 @@ exec java \
     -Djava.io.tmpdir=/tmp \
     -Dlog4j2.configurationFile=resources/log4j2.xml \
     -Dadmin.lock=${GLUU_CASA_ADMIN_LOCK_FILE} \
+    $(get_prometheus_opt) \
     ${CN_JAVA_OPTIONS} \
     -jar /opt/jetty/start.jar jetty.httpConfig.sendServerVersion=false jetty.deploy.scanInterval=0
diff --git a/...n-setup/pygluu/kubernetes/templates/helm/gluu/charts/auth-server/templates/deployment.yml b/...n-setup/pygluu/kubernetes/templates/helm/gluu/charts/auth-server/templates/deployment.yml
@@ -59,6 +59,10 @@ spec:
         ports:
         - name: {{ .Values.service.name }}
           containerPort: {{ .Values.service.port }}
+        {{ if .Values.global.cnPrometheusPort }}
+        - name: prometheus-port
+          containerPort: {{ .Values.global.cnPrometheusPort }}
+        {{- end }}
         envFrom:
         - configMapRef:
             name: {{ .Release.Name }}-config-cm

diff --git a/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/casa/templates/deployment.yaml b/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/casa/templates/deployment.yaml
@@ -64,6 +64,10 @@ spec:
             - name: {{ .Values.service.name }}
               containerPort: {{ .Values.service.port}}
               protocol: TCP
+            {{ if .Values.global.cnPrometheusPort }}
+            - name: prometheus-port
+              containerPort: {{ .Values.global.cnPrometheusPort }}
+            {{- end }}
           envFrom:
             - configMapRef: 
                 name: {{ .Release.Name }}-config-cm

diff --git a/...n-setup/pygluu/kubernetes/templates/helm/gluu/charts/client-api/templates/deployment.yaml b/...n-setup/pygluu/kubernetes/templates/helm/gluu/charts/client-api/templates/deployment.yaml
@@ -61,6 +61,10 @@ spec:
           ports:
             - containerPort: 8444
             - containerPort: 8443
+            {{ if .Values.global.cnPrometheusPort }}
+            - name: prometheus-port
+              containerPort: {{ .Values.global.cnPrometheusPort }}
+            {{- end }}
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-config-cm

diff --git a/...n-setup/pygluu/kubernetes/templates/helm/gluu/charts/config-api/templates/deployment.yaml b/...n-setup/pygluu/kubernetes/templates/helm/gluu/charts/config-api/templates/deployment.yaml
@@ -56,6 +56,10 @@ spec:
           ports:
             - containerPort: 9444
             - containerPort: 8074
+            {{ if .Values.global.cnPrometheusPort }}
+            - name: prometheus-port
+              containerPort: {{ .Values.global.cnPrometheusPort }}
+            {{- end }}
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-config-cm

diff --git a/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/config/templates/configmaps.yaml b/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/config/templates/configmaps.yaml
@@ -17,6 +17,8 @@ metadata:
 data:
   # Jetty header size in bytes in the auth server
   CN_JETTY_REQUEST_HEADER_SIZE: {{ .Values.configmap.cnJettyRequestHeaderSize | quote }}
+  # Port used by Prometheus JMX agent
+  CN_PROMETHEUS_PORT: {{ .Values.global.cnPrometheusPort }}
   CN_DISTRIBUTION: {{ .Values.global.distribution | quote }}
   {{ if .Values.global.cnObExtSigningJwksUri }}
   CN_OB_EXT_SIGNING_JWKS_URI: {{ .Values.global.cnObExtSigningJwksUri | quote }}

diff --git a/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/fido2/templates/deployment.yml b/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/fido2/templates/deployment.yml
@@ -59,6 +59,10 @@ spec:
         ports:
         - name: {{ .Values.service.name }}
           containerPort: {{ .Values.service.port }}
+        {{ if .Values.global.cnPrometheusPort }}
+        - name: prometheus-port
+          containerPort: {{ .Values.global.cnPrometheusPort }}
+        {{- end }}
         envFrom:
         - configMapRef:
             name: {{ .Release.Name }}-config-cm

diff --git a/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/scim/templates/deployment.yml b/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/charts/scim/templates/deployment.yml
@@ -67,6 +67,10 @@ spec:
         ports:
         - name: {{ .Values.service.name }}
           containerPort: {{ .Values.service.port }}
+        {{ if .Values.global.cnPrometheusPort }}
+        - name: prometheus-port
+          containerPort: {{ .Values.global.cnPrometheusPort }}
+        {{- end }}
         envFrom:
         - configMapRef:
             name: {{ .Release.Name }}-config-cm

diff --git a/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/values.yaml b/flex-cn-setup/pygluu/kubernetes/templates/helm/gluu/values.yaml
@@ -823,6 +823,8 @@ global:
   cloud:
     # -- Boolean flag if enabled will strip resources requests and limits from all services.
     testEnviroment: false
+  # -- Port used by Prometheus JMX agent (default to empty string). To enable Prometheus JMX agent, set the value to a number.
+  cnPrometheusPort: ""
   # -- Document store type to use for shibboleth files LOCAL.
   cnDocumentStoreType: LOCAL
   # -- Persistence backend to run Gluu with ldap|couchbase|hybrid|sql|spanner.