feat: introduce new hybrid persistence mapping (#250)

* feat: introduce new hybrid persistence mapping * build: update jans-pycloudlib version
GluuFederation · Jun 8, 2022 · 473abdc · 473abdc
1 parent 759e3db
commit 473abdc
Show file tree

Hide file tree

Showing 10 changed files with 148 additions and 58 deletions.
diff --git a/docker-admin-ui/Dockerfile b/docker-admin-ui/Dockerfile
@@ -35,7 +35,7 @@ RUN python3 -m ensurepip \
 # jans-linux-setup sync
 # =====================
 
-ENV JANS_LINUX_SETUP_VERSION=eb113d09421b95671fe1ab4eaa5c4bafc2aed6af
+ENV JANS_LINUX_SETUP_VERSION=afc539d63a70cda21e990fecd470224a4f94e9d6
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 
 # note that as we're pulling from a monorepo (with multiple project in it)
@@ -65,7 +65,7 @@ RUN cd /tmp/jans \
 # TODO:
 # - use NODE_ENV=production
 # - download build package (not git clone)
-ENV ADMIN_UI_VERSION=e042a9de63fba051535782b242f38843d39f5afd
+ENV ADMIN_UI_VERSION=9de53ada7bd2ca576e8b64b09fa51b002223e84e
 
 # note that as we're pulling from a monorepo (with multiple project in it)
 # we are using partial-clone and sparse-checkout to get the admin-ui code
@@ -140,6 +140,27 @@ ENV CN_SECRET_ADAPTER=vault \
     CN_SECRET_GOOGLE_SECRET_VERSION_ID=latest \
     CN_SECRET_GOOGLE_SECRET_NAME_PREFIX=jans
 
+# ===============
+# Persistence ENV
+# ===============
+
+ENV CN_PERSISTENCE_TYPE=ldap \
+    CN_HYBRID_MAPPING="{}" \
+    CN_LDAP_URL=localhost:1636 \
+    CN_LDAP_USE_SSL=true \
+    CN_COUCHBASE_URL=localhost \
+    CN_COUCHBASE_USER=admin \
+    CN_COUCHBASE_CERT_FILE=/etc/certs/couchbase.crt \
+    CN_COUCHBASE_CONN_TIMEOUT=10000 \
+    CN_COUCHBASE_CONN_MAX_WAIT=20000 \
+    CN_COUCHBASE_SCAN_CONSISTENCY=not_bounded \
+    CN_COUCHBASE_BUCKET_PREFIX=jans \
+    CN_COUCHBASE_TRUSTSTORE_ENABLE=true \
+    CN_COUCHBASE_KEEPALIVE_INTERVAL=30000 \
+    CN_COUCHBASE_KEEPALIVE_TIMEOUT=2500 \
+    CN_GOOGLE_SPANNER_INSTANCE_ID="" \
+    CN_GOOGLE_SPANNER_DATABASE_ID=""
+
 # ===========
 # Generic ENV
 # ===========

diff --git a/docker-admin-ui/README.md b/docker-admin-ui/README.md
@@ -53,3 +53,63 @@ The following environment variables are supported by the container:
 - `CN_TOKEN_SERVER_USERINFO_ENDPOINT`: User info endpoint at token server (default to `/jans-auth/restv1/userinfo`).
 - `CN_TOKEN_SERVER_CLIENT_ID`: Client ID registered at token server.
 - `CN_TOKEN_SERVER_CERT_FILE`: Path to token server certificate (default to `/etc/certs/token_server.crt`).
+- `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`).
+- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details.
+- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`).
+- `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`).
+- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`).
+- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`).
+- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`).
+- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`).
+- `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds).
+- `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds).
+- `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`).
+- `CN_COUCHBASE_BUCKET_PREFIX`: Prefix for Couchbase buckets (default to `jans`).
+- `CN_COUCHBASE_TRUSTSTORE_ENABLE`: Enable truststore for encrypted Couchbase connection (default to `true`).
+- `CN_COUCHBASE_KEEPALIVE_INTERVAL`: Keep-alive interval for Couchbase connection (default to `30000` milliseconds).
+- `CN_COUCHBASE_KEEPALIVE_TIMEOUT`: Keep-alive timeout for Couchbase connection (default to `2500` milliseconds).
+- `CN_SQL_DB_DIALECT`: Dialect name of SQL backend (one of `mysql`, `pgsql`; default to `mysql`).
+- `CN_SQL_DB_HOST`: Host of SQL backend (default to `localhost`).
+- `CN_SQL_DB_PORT`: Port of SQL backend (default to `3306`).
+- `CN_SQL_DB_NAME`: Database name (default to `jans`)
+- `CN_SQL_DB_USER`: User name to interact with SQL backend (default to `jans`).
+- `CN_GOOGLE_SPANNER_INSTANCE_ID`: Instance ID of Google Spanner (default to empty string).
+- `CN_GOOGLE_SPANNER_DATABASE_ID`: Database ID of Google Spanner (default to empty string).
+- `GOOGLE_APPLICATION_CREDENTIALS`: Path to Google credentials JSON file (default to `/etc/jans/conf/google-credentials.json`).
+- `GOOGLE_PROJECT_ID`: Google Project ID (default to empty string).
+- `GOOGLE_PROJECT_ID`: Google Project ID (default to empty string). Used when `CN_CONFIG_ADAPTER` or `CN_SECRET_ADAPTER` set to `google`.
+- `GOOGLE_APPLICATION_CREDENTIALS`: Path to Google credentials JSON file (default to `/etc/jans/conf/google-credentials.json`). Used when `CN_CONFIG_ADAPTER` or `CN_SECRET_ADAPTER` set to `google`.
+- `CN_GOOGLE_SPANNER_INSTANCE_ID`: Google Spanner instance ID.
+- `CN_GOOGLE_SPANNER_DATABASE_ID`: Google Spanner database ID.
+
+### Hybrid mapping
+
+Hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below:
+
+1.  Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid`
+
+1.  Set `CN_HYBRID_MAPPING` with the following format:
+
+    ```
+    {
+        "default": "<couchbase|ldap|spanner|sql>",
+        "user": "<couchbase|ldap|spanner|sql>",
+        "site": "<couchbase|ldap|spanner|sql>",
+        "cache": "<couchbase|ldap|spanner|sql>",
+        "token": "<couchbase|ldap|spanner|sql>",
+        "session": "<couchbase|ldap|spanner|sql>",
+    }
+    ```
+
+    Example:
+
+    ```
+    {
+        "default": "sql",
+        "user": "spanner",
+        "site": "ldap",
+        "cache": "sql",
+        "token": "couchbase",
+        "session": "spanner",
+    }
+    ```
diff --git a/docker-admin-ui/requirements.txt b/docker-admin-ui/requirements.txt
@@ -1,4 +1,4 @@
 libcst<0.4
 # pinned to py3-grpcio version to avoid failure on native extension build
 grpcio==1.41.0
-git+https://github.com/JanssenProject/jans@f2e653ef917efd017195f2330b64e64c333f4699#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans@d9f9cd6bb55a644d60eba7355a5077f159854020#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-admin-ui/scripts/bootstrap.py b/docker-admin-ui/scripts/bootstrap.py
@@ -10,6 +10,7 @@
 from jans.pycloudlib.persistence import LdapClient
 from jans.pycloudlib.persistence import SpannerClient
 from jans.pycloudlib.persistence import SqlClient
+from jans.pycloudlib.persistence.utils import PersistenceMapper
 
 from settings import LOGGING_CONFIG
 
@@ -82,16 +83,8 @@ def __init__(self, manager):
         }
 
         # determine persistence type
-        self.persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
-        ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
-
-        if self.persistence_type == "hybrid":
-            if ldap_mapping == "default":
-                client_cls = LdapClient
-                self.persistence_type = "ldap"
-            else:
-                client_cls = CouchbaseClient
-                self.persistence_type = "couchbase"
+        mapper = PersistenceMapper()
+        self.persistence_type = mapper.mapping["default"]
 
         # determine persistence client
         client_cls = client_classes.get(self.persistence_type)

diff --git a/docker-admin-ui/scripts/wait.py b/docker-admin-ui/scripts/wait.py
@@ -1,39 +1,33 @@
-import logging
 import logging.config
 import os
 
 from jans.pycloudlib import get_manager
 from jans.pycloudlib import wait_for
+from jans.pycloudlib import wait_for_persistence
 from jans.pycloudlib.validators import validate_persistence_type
-from jans.pycloudlib.validators import validate_persistence_ldap_mapping
+from jans.pycloudlib.validators import validate_persistence_hybrid_mapping
 from jans.pycloudlib.validators import validate_persistence_sql_dialect
 
 from settings import LOGGING_CONFIG
 
 logging.config.dictConfig(LOGGING_CONFIG)
-logger = logging.getLogger("wait")
 
 
 def main():
     persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
     validate_persistence_type(persistence_type)
 
-    ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
-    validate_persistence_ldap_mapping(persistence_type, ldap_mapping)
+    if persistence_type == "hybrid":
+        validate_persistence_hybrid_mapping()
 
     if persistence_type == "sql":
         sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql")
         validate_persistence_sql_dialect(sql_dialect)
 
     manager = get_manager()
     deps = ["config", "secret"]
-
-    if persistence_type == "hybrid":
-        deps += ["ldap", "couchbase"]
-    else:
-        deps.append(persistence_type)
-
     wait_for(manager, deps)
+    wait_for_persistence(manager)
 
 
 if __name__ == "__main__":

diff --git a/docker-casa/Dockerfile b/docker-casa/Dockerfile
@@ -35,7 +35,7 @@ EXPOSE 8080
 # ====
 
 ENV GLUU_VERSION=5.0.0-SNAPSHOT
-ENV GLUU_BUILD_DATE='2022-05-12 11:53'
+ENV GLUU_BUILD_DATE='2022-05-26 13:57'
 ENV GLUU_SOURCE_URL=https://jenkins.gluu.org/maven/org/gluu/casa/${GLUU_VERSION}/casa-${GLUU_VERSION}.war
 
 # Install Casa
@@ -63,7 +63,7 @@ RUN python3 -m ensurepip \
 # jans-linux-setup sync
 # =====================
 
-ENV JANS_LINUX_SETUP_VERSION=eb113d09421b95671fe1ab4eaa5c4bafc2aed6af
+ENV JANS_LINUX_SETUP_VERSION=afc539d63a70cda21e990fecd470224a4f94e9d6
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 
 # note that as we're pulling from a monorepo (with multiple project in it)
@@ -90,7 +90,7 @@ RUN cd /tmp/jans \
 # flex setup sync
 # ===============
 
-ENV FLEX_VERSION=e042a9de63fba051535782b242f38843d39f5afd
+ENV FLEX_VERSION=9de53ada7bd2ca576e8b64b09fa51b002223e84e
 ARG FLEX_SETUP_DIR=flex-linux-setup/flex_linux_setup
 ARG CASA_EXTRAS_DIR=casa/extras
 
@@ -168,7 +168,7 @@ ENV CN_SECRET_ADAPTER=vault \
 # ===============
 
 ENV CN_PERSISTENCE_TYPE=ldap \
-    CN_PERSISTENCE_LDAP_MAPPING=default \
+    CN_HYBRID_MAPPING="{}" \
     CN_LDAP_URL=localhost:1636 \
     CN_LDAP_USE_SSL=true \
     CN_COUCHBASE_URL=localhost \

diff --git a/docker-casa/README.md b/docker-casa/README.md
@@ -41,7 +41,7 @@ The following environment variables are supported by the container:
 - `CN_WAIT_SLEEP_DURATION`: Delay between startup "health checks" (default to `10` seconds).
 - `CN_MAX_RAM_PERCENTAGE`: Value passed to Java option `-XX:MaxRAMPercentage`.
 - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`).
-- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, `token`, or `session`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`.
+- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details.
 - `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`.
 - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`).
 - `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`.
@@ -102,3 +102,35 @@ The following key-value pairs are the defaults:
     "timer_log_level": "INFO"
 }
 ```
+
+### Hybrid mapping
+
+Hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below:
+
+1.  Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid`
+
+1.  Set `CN_HYBRID_MAPPING` with the following format:
+
+    ```
+    {
+        "default": "<couchbase|ldap|spanner|sql>",
+        "user": "<couchbase|ldap|spanner|sql>",
+        "site": "<couchbase|ldap|spanner|sql>",
+        "cache": "<couchbase|ldap|spanner|sql>",
+        "token": "<couchbase|ldap|spanner|sql>",
+        "session": "<couchbase|ldap|spanner|sql>",
+    }
+    ```
+
+    Example:
+
+    ```
+    {
+        "default": "sql",
+        "user": "spanner",
+        "site": "ldap",
+        "cache": "sql",
+        "token": "couchbase",
+        "session": "spanner",
+    }
+    ```
diff --git a/docker-casa/requirements.txt b/docker-casa/requirements.txt
@@ -2,4 +2,4 @@ webdavclient3>=3.14.5
 libcst<0.4
 # pinned to py3-grpcio version to avoid failure on native extension build
 grpcio==1.41.0
-git+https://github.com/JanssenProject/jans@f2e653ef917efd017195f2330b64e64c333f4699#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans@d9f9cd6bb55a644d60eba7355a5077f159854020#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-casa/scripts/bootstrap.py b/docker-casa/scripts/bootstrap.py
@@ -23,6 +23,7 @@
 from jans.pycloudlib.persistence import SqlClient
 from jans.pycloudlib.persistence import doc_id_from_dn
 from jans.pycloudlib.persistence import id_from_dn
+from jans.pycloudlib.persistence.utils import PersistenceMapper
 from jans.pycloudlib.utils import cert_to_truststore
 from jans.pycloudlib.utils import get_random_chars
 from jans.pycloudlib.utils import encode_text
@@ -141,33 +142,36 @@ def main():
     render_salt(manager, "/app/templates/salt.tmpl", "/etc/jans/conf/salt")
     render_base_properties("/app/templates/jans.properties.tmpl", "/etc/jans/conf/jans.properties")
 
-    if persistence_type in ("ldap", "hybrid"):
+    mapper = PersistenceMapper()
+    persistence_groups = mapper.groups()
+
+    if persistence_type == "hybrid":
+        render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties")
+
+    if "ldap" in persistence_groups:
         render_ldap_properties(
             manager,
             "/app/templates/jans-ldap.properties.tmpl",
             "/etc/jans/conf/jans-ldap.properties",
         )
         sync_ldap_truststore(manager)
 
-    if persistence_type in ("couchbase", "hybrid"):
+    if "couchbase" in persistence_groups:
         render_couchbase_properties(
             manager,
             "/app/templates/jans-couchbase.properties.tmpl",
             "/etc/jans/conf/jans-couchbase.properties",
         )
         sync_couchbase_truststore(manager)
 
-    if persistence_type == "hybrid":
-        render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties")
-
-    if persistence_type == "sql":
+    if "sql" in persistence_groups:
         render_sql_properties(
             manager,
             "/app/templates/jans-sql.properties.tmpl",
             "/etc/jans/conf/jans-sql.properties",
         )
 
-    if persistence_type == "spanner":
+    if "spanner" in persistence_groups:
         render_spanner_properties(
             manager,
             "/app/templates/jans-spanner.properties.tmpl",
@@ -204,16 +208,8 @@ def __init__(self, manager):
         }
 
         # determine persistence type
-        self.persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
-        ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
-
-        if self.persistence_type == "hybrid":
-            if ldap_mapping == "default":
-                client_cls = LdapClient
-                self.persistence_type = "ldap"
-            else:
-                client_cls = CouchbaseClient
-                self.persistence_type = "couchbase"
+        mapper = PersistenceMapper()
+        self.persistence_type = mapper.mapping["default"]
 
         # determine persistence client
         client_cls = client_classes.get(self.persistence_type)

diff --git a/docker-casa/scripts/wait.py b/docker-casa/scripts/wait.py
@@ -1,39 +1,33 @@
-import logging
 import logging.config
 import os
 
 from jans.pycloudlib import get_manager
 from jans.pycloudlib import wait_for
+from jans.pycloudlib import wait_for_persistence
 from jans.pycloudlib.validators import validate_persistence_type
-from jans.pycloudlib.validators import validate_persistence_ldap_mapping
+from jans.pycloudlib.validators import validate_persistence_hybrid_mapping
 from jans.pycloudlib.validators import validate_persistence_sql_dialect
 
 from settings import LOGGING_CONFIG
 
 logging.config.dictConfig(LOGGING_CONFIG)
-logger = logging.getLogger("wait")
 
 
 def main():
     persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
     validate_persistence_type(persistence_type)
 
-    ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
-    validate_persistence_ldap_mapping(persistence_type, ldap_mapping)
+    if persistence_type == "hybrid":
+        validate_persistence_hybrid_mapping()
 
     if persistence_type == "sql":
         sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql")
         validate_persistence_sql_dialect(sql_dialect)
 
     manager = get_manager()
     deps = ["config", "secret"]
-
-    if persistence_type == "hybrid":
-        deps += ["ldap", "couchbase"]
-    else:
-        deps.append(persistence_type)
-
     wait_for(manager, deps)
+    wait_for_persistence(manager)
 
 
 if __name__ == "__main__":