From 0b6334acdb862ce458c628a8eb81ef0b8f7c5dcb Mon Sep 17 00:00:00 2001 From: Isman Firmansyah Date: Fri, 11 Mar 2022 19:25:34 +0700 Subject: [PATCH] feat: add support to import custom ldif (#1002) * refactor: remove unused initialization check * feat(ldap): add support for importing custom LDIF files * feat(sql): add support for importing custom LDIF files * feat(spanner): add support for importing custom LDIF files * refactor: put generated ldif template under /app/tmp * feat(couchbase): add support for importing custom LDIF files * docs: add custom ldif instructions Co-authored-by: Mohammad Abudayyeh <47318409+moabu@users.noreply.github.com> --- docker-jans-persistence-loader/Dockerfile | 8 +- .../scripts/couchbase_setup.py | 110 ++++++++++++------ .../scripts/ldap_setup.py | 86 ++++++-------- .../scripts/spanner_setup.py | 52 ++++++--- .../scripts/sql_setup.py | 47 +++++--- docs/user/how-to/add-custom-ldifs.md | 79 +++++++++++++ 6 files changed, 267 insertions(+), 115 deletions(-) create mode 100644 docs/user/how-to/add-custom-ldifs.md diff --git a/docker-jans-persistence-loader/Dockerfile b/docker-jans-persistence-loader/Dockerfile index 537b3602e19..d24eac27c82 100644 --- a/docker-jans-persistence-loader/Dockerfile +++ b/docker-jans-persistence-loader/Dockerfile @@ -177,23 +177,27 @@ LABEL name="Persistence" \ summary="Janssen Authorization Server Persistence loader" \ description="Generate initial data for persistence layer" -RUN mkdir -p /app/tmp /etc/certs /etc/jans/conf +RUN mkdir -p /app/tmp /app/custom_ldif /etc/certs /etc/jans/conf COPY scripts /app/scripts # this overrides existing templates COPY templates /app/templates RUN chmod +x /app/scripts/entrypoint.sh -# # create non-root user +# create non-root user RUN adduser -s /bin/sh -D -G root -u 1000 1000 # adjust ownership RUN chown -R 1000:1000 /tmp \ && chown -R 1000:1000 /app/tmp/ \ + && chown -R 1000:1000 /app/custom_ldif/ \ && chgrp -R 0 /tmp && chmod -R g=u /tmp \ && chgrp -R 0 /app/tmp && chmod -R g=u /app/tmp \ + && chgrp -R 0 /app/custom_ldif && chmod -R g=u /app/custom_ldif \ && chgrp -R 0 /etc/certs && chmod -R g=u /etc/certs \ && chgrp -R 0 /etc/jans && chmod -R g=u /etc/jans + USER 1000 + ENTRYPOINT ["tini", "-g", "--"] CMD ["sh", "/app/scripts/entrypoint.sh"] diff --git a/docker-jans-persistence-loader/scripts/couchbase_setup.py b/docker-jans-persistence-loader/scripts/couchbase_setup.py index a2b3ce7ca0c..9413936fc4b 100644 --- a/docker-jans-persistence-loader/scripts/couchbase_setup.py +++ b/docker-jans-persistence-loader/scripts/couchbase_setup.py @@ -4,6 +4,7 @@ import logging.config import os import time +from pathlib import Path from ldif import LDIFParser @@ -29,32 +30,32 @@ def get_bucket_mappings(manager): "default": { "bucket": prefix, "mem_alloc": 100, - "document_key_prefix": [], + # "document_key_prefix": [], }, "user": { "bucket": f"{prefix}_user", "mem_alloc": 300, - "document_key_prefix": ["groups_", "people_", "authorizations_"], + # "document_key_prefix": ["groups_", "people_", "authorizations_"], }, "site": { "bucket": f"{prefix}_site", "mem_alloc": 100, - "document_key_prefix": ["site_", "cache-refresh_"], + # "document_key_prefix": ["site_", "cache-refresh_"], }, "token": { "bucket": f"{prefix}_token", "mem_alloc": 300, - "document_key_prefix": ["tokens_"], + # "document_key_prefix": ["tokens_"], }, "cache": { "bucket": f"{prefix}_cache", "mem_alloc": 100, - "document_key_prefix": ["cache_"], + # "document_key_prefix": ["cache_"], }, "session": { "bucket": f"{prefix}_session", "mem_alloc": 200, - "document_key_prefix": [], + # "document_key_prefix": [], }, } @@ -205,6 +206,7 @@ def __init__(self, manager): self.client = CouchbaseClient(hostname, user, password) self.manager = manager self.index_num_replica = 0 + self.attr_processor = AttrProcessor() def create_buckets(self, bucket_mappings, bucket_type="couchbase"): sys_info = self.client.get_system_info() @@ -326,37 +328,10 @@ def create_indexes(self, bucket_mappings): continue logger.warning("Failed to execute query, reason={}".format(error["msg"])) - def import_ldif(self, bucket_mappings): - ctx = prepare_template_ctx(self.manager) - attr_processor = AttrProcessor() - + def import_builtin_ldif(self, bucket_mappings, ctx): for _, mapping in bucket_mappings.items(): for file_ in mapping["files"]: - logger.info(f"Importing {file_} file") - src = f"/app/templates/{file_}" - dst = f"/app/tmp/{file_}" - os.makedirs(os.path.dirname(dst), exist_ok=True) - - render_ldif(src, dst, ctx) - - with open(dst, "rb") as fd: - parser = LDIFParser(fd) - - for dn, entry in parser.parse(): - if len(entry) <= 2: - continue - - key = id_from_dn(dn) - entry["dn"] = [dn] - entry = transform_entry(entry, attr_processor) - data = json.dumps(entry) - - # using INSERT will cause duplication error, but the data is left intact - query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (mapping["bucket"], key, data) - req = self.client.exec_query(query) - - if not req.ok: - logger.warning("Failed to execute query, reason={}".format(req.json())) + self._import_ldif(f"/app/templates/{file_}", ctx) def initialize(self): num_replica = int(os.environ.get("CN_COUCHBASE_INDEX_NUM_REPLICA", 0)) @@ -376,7 +351,9 @@ def initialize(self): self.create_indexes(bucket_mappings) time.sleep(5) - self.import_ldif(bucket_mappings) + ctx = prepare_template_ctx(self.manager) + self.import_builtin_ldif(bucket_mappings, ctx) + self.import_custom_ldif(ctx) time.sleep(5) self.create_couchbase_shib_user() @@ -388,3 +365,64 @@ def create_couchbase_shib_user(self): 'Shibboleth IDP', 'query_select[*]', ) + + def import_custom_ldif(self, ctx): + custom_dir = Path("/app/custom_ldif") + + for file_ in custom_dir.rglob("*.ldif"): + self._import_ldif(file_, ctx) + + def _import_ldif(self, path, ctx): + src = Path(path).resolve() + + # generated template will be saved under ``/app/tmp`` directory + # examples: + # - ``/app/templates/groups.ldif`` will be saved as ``/app/tmp/templates/groups.ldif`` + # - ``/app/custom_ldif/groups.ldif`` will be saved as ``/app/tmp/custom_ldif/groups.ldif`` + dst = Path("/app/tmp").joinpath(str(src).removeprefix("/app/")).resolve() + + # ensure directory for generated template is exist + dst.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Importing {src} file") + render_ldif(src, dst, ctx) + + with open(dst, "rb") as fd: + parser = LDIFParser(fd) + + for dn, entry in parser.parse(): + if len(entry) <= 2: + continue + + key = id_from_dn(dn) + bucket = get_bucket_for_key(key) + entry["dn"] = [dn] + entry = transform_entry(entry, self.attr_processor) + data = json.dumps(entry) + + # TODO: get the bucket based on key prefix + # using INSERT will cause duplication error, but the data is left intact + query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (bucket, key, data) + req = self.client.exec_query(query) + + if not req.ok: + logger.warning("Failed to execute query, reason={}".format(req.json())) + + +def get_bucket_for_key(key): + bucket_prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") + + cursor = key.find("_") + key_prefix = key[:cursor + 1] + + if key_prefix in ("groups_", "people_", "authorizations_"): + bucket = f"{bucket_prefix}_user" + elif key_prefix in ("site_", "cache-refresh_"): + bucket = f"{bucket_prefix}_site" + elif key_prefix in ("tokens_"): + bucket = f"{bucket_prefix}_token" + elif key_prefix in ("cache_"): + bucket = f"{bucket_prefix}_cache" + else: + bucket = bucket_prefix + return bucket diff --git a/docker-jans-persistence-loader/scripts/ldap_setup.py b/docker-jans-persistence-loader/scripts/ldap_setup.py index 4170b14251a..84f6314864c 100644 --- a/docker-jans-persistence-loader/scripts/ldap_setup.py +++ b/docker-jans-persistence-loader/scripts/ldap_setup.py @@ -2,13 +2,13 @@ import logging.config import os import time +from pathlib import Path from ldap3.core.exceptions import LDAPSessionTerminatedByServerError from ldap3.core.exceptions import LDAPSocketOpenError from ldif import LDIFParser -from jans.pycloudlib.utils import as_boolean from jans.pycloudlib.persistence.ldap import LdapClient from settings import LOGGING_CONFIG @@ -54,7 +54,7 @@ def check_indexes(self, mapping): "retrying in {} seconds".format(reason, sleep_duration)) time.sleep(sleep_duration) - def import_ldif(self): + def import_builtin_ldif(self, ctx): optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]")) ldif_mappings = get_ldif_mappings(optional_scopes) @@ -65,31 +65,19 @@ def import_ldif(self): mapping = ldap_mapping ldif_mappings = {mapping: ldif_mappings[mapping]} - # # these mappings require `base.ldif` + # these mappings require `base.ldif` # opt_mappings = ("user", "token",) - # `user` mapping requires `o=gluu` which available in `base.ldif` + # `user` mapping requires `o=jans` which available in `base.ldif` # if mapping in opt_mappings and "base.ldif" not in ldif_mappings[mapping]: if "base.ldif" not in ldif_mappings[mapping]: ldif_mappings[mapping].insert(0, "base.ldif") - ctx = prepare_template_ctx(self.manager) - for mapping, files in ldif_mappings.items(): self.check_indexes(mapping) for file_ in files: - logger.info(f"Importing {file_} file") - src = f"/app/templates/{file_}" - dst = f"/app/tmp/{file_}" - os.makedirs(os.path.dirname(dst), exist_ok=True) - - render_ldif(src, dst, ctx) - - with open(dst, "rb") as fd: - parser = LDIFParser(fd) - for dn, entry in parser.parse(): - self.add_entry(dn, entry) + self._import_ldif(f"/app/templates/{file_}", ctx) def add_entry(self, dn, attrs): max_wait_time = 300 @@ -106,34 +94,36 @@ def add_entry(self, dn, attrs): time.sleep(sleep_duration) def initialize(self): - def is_initialized(): - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - # a minimum service stack is having oxTrust, hence check whether entry - # for oxTrust exists in LDAP - default_search = ("ou=jans-auth,ou=configuration,o=jans", - "(objectClass=jansAppConf)") - - if persistence_type == "hybrid": - # `cache` and `token` mapping only have base entries - search_mapping = { - "default": default_search, - "user": ("inum=60B7,ou=groups,o=jans", "(objectClass=jansGrp)"), - "site": ("ou=cache-refresh,o=site", "(ou=people)"), - "cache": ("o=jans", "(ou=cache)"), - "token": ("ou=tokens,o=jans", "(ou=tokens)"), - "session": ("ou=sessions,o=jans", "(ou=sessions)"), - } - search = search_mapping[ldap_mapping] - else: - search = default_search - return self.client.search(search[0], search[1], attributes=["objectClass"], limit=1) - - should_skip = as_boolean( - os.environ.get("CN_PERSISTENCE_SKIP_INITIALIZED", False), - ) - if should_skip and is_initialized(): - logger.info("LDAP backend already initialized") - return - self.import_ldif() + ctx = prepare_template_ctx(self.manager) + + logger.info("Importing builtin LDIF files") + self.import_builtin_ldif(ctx) + + logger.info("Importing custom LDIF files (if any)") + self.import_custom_ldif(ctx) + + def import_custom_ldif(self, ctx): + custom_dir = Path("/app/custom_ldif") + + for file_ in custom_dir.rglob("*.ldif"): + self._import_ldif(file_, ctx) + + def _import_ldif(self, path, ctx): + src = Path(path).resolve() + + # generated template will be saved under ``/app/tmp`` directory + # examples: + # - ``/app/templates/groups.ldif`` will be saved as ``/app/tmp/templates/groups.ldif`` + # - ``/app/custom_ldif/groups.ldif`` will be saved as ``/app/tmp/custom_ldif/groups.ldif`` + dst = Path("/app/tmp").joinpath(str(src).removeprefix("/app/")).resolve() + + # ensure directory for generated template is exist + dst.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Importing {src} file") + render_ldif(src, dst, ctx) + + with open(dst, "rb") as fd: + parser = LDIFParser(fd) + for dn, entry in parser.parse(): + self.add_entry(dn, entry) diff --git a/docker-jans-persistence-loader/scripts/spanner_setup.py b/docker-jans-persistence-loader/scripts/spanner_setup.py index 06c4c91aca1..ca076766f7c 100644 --- a/docker-jans-persistence-loader/scripts/spanner_setup.py +++ b/docker-jans-persistence-loader/scripts/spanner_setup.py @@ -2,10 +2,10 @@ import hashlib import json import logging.config -import os import re from collections import OrderedDict from collections import defaultdict +from pathlib import Path from ldif import LDIFParser @@ -232,25 +232,13 @@ def create_indexes(self): # run the callback self.create_spanner_indexes(table_name, column_mapping) - def import_ldif(self): + def import_builtin_ldif(self, ctx): optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]")) ldif_mappings = get_ldif_mappings(optional_scopes) - ctx = prepare_template_ctx(self.manager) - for _, files in ldif_mappings.items(): for file_ in files: - logger.info(f"Importing {file_} file") - src = f"/app/templates/{file_}" - dst = f"/app/tmp/{file_}" - os.makedirs(os.path.dirname(dst), exist_ok=True) - - render_ldif(src, dst, ctx) - - for table_name, column_mapping in self.data_from_ldif(dst): - self.client.insert_into(table_name, column_mapping) - # inject rows into subtable (if any) - self.insert_into_subtable(table_name, column_mapping) + self._import_ldif(f"/app/templates/{file_}", ctx) def initialize(self): logger.info("Creating tables (if not exist)") @@ -263,7 +251,13 @@ def initialize(self): logger.info("Creating indexes (if not exist)") self.create_indexes() - self.import_ldif() + ctx = prepare_template_ctx(self.manager) + + logger.info("Importing builtin LDIF files") + self.import_builtin_ldif(ctx) + + logger.info("Importing custom LDIF files (if any)") + self.import_custom_ldif(ctx) def transform_value(self, key, values): type_ = self.sql_data_types.get(key) @@ -555,3 +549,29 @@ def column_from_array(table_name, col_name): ("jansPerson", "jansOTPDevices"), ]: column_from_array(mod[0], mod[1]) + + def import_custom_ldif(self, ctx): + custom_dir = Path("/app/custom_ldif") + + for file_ in custom_dir.rglob("*.ldif"): + self._import_ldif(file_, ctx) + + def _import_ldif(self, path, ctx): + src = Path(path).resolve() + + # generated template will be saved under ``/app/tmp`` directory + # examples: + # - ``/app/templates/groups.ldif`` will be saved as ``/app/tmp/templates/groups.ldif`` + # - ``/app/custom_ldif/groups.ldif`` will be saved as ``/app/tmp/custom_ldif/groups.ldif`` + dst = Path("/app/tmp").joinpath(str(src).removeprefix("/app/")).resolve() + + # ensure directory for generated template is exist + dst.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Importing {src} file") + render_ldif(src, dst, ctx) + + for table_name, column_mapping in self.data_from_ldif(dst): + self.client.insert_into(table_name, column_mapping) + # inject rows into subtable (if any) + self.insert_into_subtable(table_name, column_mapping) diff --git a/docker-jans-persistence-loader/scripts/sql_setup.py b/docker-jans-persistence-loader/scripts/sql_setup.py index 973b7bb64a5..29e73009a1c 100644 --- a/docker-jans-persistence-loader/scripts/sql_setup.py +++ b/docker-jans-persistence-loader/scripts/sql_setup.py @@ -6,6 +6,7 @@ from collections import OrderedDict from collections import defaultdict from string import Template +from pathlib import Path from ldif import LDIFParser @@ -238,23 +239,13 @@ def create_indexes(self): # run the callback index_func(table_name, column_mapping) - def import_ldif(self): + def import_builtin_ldif(self, ctx): optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]")) ldif_mappings = get_ldif_mappings(optional_scopes) - ctx = prepare_template_ctx(self.manager) - for _, files in ldif_mappings.items(): for file_ in files: - logger.info(f"Importing {file_} file") - src = f"/app/templates/{file_}" - dst = f"/app/tmp/{file_}" - os.makedirs(os.path.dirname(dst), exist_ok=True) - - render_ldif(src, dst, ctx) - - for table_name, column_mapping in self.data_from_ldif(dst): - self.client.insert_into(table_name, column_mapping) + self._import_ldif(f"/app/templates/{file_}", ctx) def initialize(self): logger.info("Creating tables (if not exist)") @@ -269,7 +260,13 @@ def initialize(self): logger.info("Creating indexes (if not exist)") self.create_indexes() - self.import_ldif() + ctx = prepare_template_ctx(self.manager) + + logger.info("Importing builtin LDIF files") + self.import_builtin_ldif(ctx) + + logger.info("Importing custom LDIF files (if any)") + self.import_custom_ldif(ctx) def transform_value(self, key, values): type_ = self.sql_data_types.get(key) @@ -499,3 +496,27 @@ def column_from_json(table_name, col_name): ("jansPerson", "jansOTPDevices"), ]: column_from_json(mod[0], mod[1]) + + def import_custom_ldif(self, ctx): + custom_dir = Path("/app/custom_ldif") + + for file_ in custom_dir.rglob("*.ldif"): + self._import_ldif(file_, ctx) + + def _import_ldif(self, path, ctx): + src = Path(path).resolve() + + # generated template will be saved under ``/app/tmp`` directory + # examples: + # - ``/app/templates/groups.ldif`` will be saved as ``/app/tmp/templates/groups.ldif`` + # - ``/app/custom_ldif/groups.ldif`` will be saved as ``/app/tmp/custom_ldif/groups.ldif`` + dst = Path("/app/tmp").joinpath(str(src).removeprefix("/app/")).resolve() + + # ensure directory for generated template is exist + dst.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Importing {src} file") + render_ldif(src, dst, ctx) + + for table_name, column_mapping in self.data_from_ldif(dst): + self.client.insert_into(table_name, column_mapping) diff --git a/docs/user/how-to/add-custom-ldifs.md b/docs/user/how-to/add-custom-ldifs.md new file mode 100644 index 00000000000..cc1b969657e --- /dev/null +++ b/docs/user/how-to/add-custom-ldifs.md @@ -0,0 +1,79 @@ +## Contents: + +- [Overview](#overview) +- [Using Kubernetes?](#kubernetes) + +## Overview + +This guide describes steps to load custom work to Janssen IDP. + + +#### Hardware configuration + +For development and POC purposes, 4GB RAM and 10 GB HDD should be available for Janssen Server. For PROD deployments, please refer [installation guide](https://github.com/JanssenProject/jans/wiki#janssen-installation). + + +#### Prerequisites +- Existing Janssen server installed + +#### Kubernetes + +If you are using Kubernetes please follow this section. You may create several ldif files denoting each type such as `custom_attributes.ldif`, `custom_clients.ldif`..etc. Refer to the built-in [`ldifs`](https://github.com/JanssenProject/jans/blob/main/jans-linux-setup/jans_setup/templates) for examples. + +1. Create your custom ldif files. These can be clients, attributes, scopes or whatever custom work you need. The example will be adding a custom attribute and we will call this file `custom_attributes.ldif`. + + ``` + dn: inum=C9B1,ou=attributes,o=jans + description: Maiden name of the End-User.Note that in some cultures, people can have multiple given names;all can be present, with the names being separated by space characters. + displayName: Maiden Name + inum: C9B1 + jansAttrEditTyp: user + jansAttrEditTyp: admin + jansAttrName: givenName + jansAttrOrigin: jansPerson + jansAttrTyp: string + jansAttrViewTyp: user + jansAttrViewTyp: admin + jansClaimName: maiden_name + jansSAML1URI: urn:mace:dir:attribute-def:maidenName + jansSAML2URI: urn:oid:2.5.4.42 + jansStatus: active + objectClass: top + objectClass: jansAttr + urn: urn:mace:dir:attribute-def:maidenName + ``` + +2. Create a configmap or secret depending on the if the ldif holds and secret data such as a client secret. Here, we will be creating a configmap. + +```bash +kubectl create cm custom-attributes -n -f custom_attributes.ldif +# kubectl create cm custom-attributes -n jans -f custom_attributes.ldif +# using a secret +# kubectl create secret generic custom-attributes -n jans -f custom_attributes.ldif +``` + +3. Mount the created configmap or secret inside your `values.yaml` + + ```yaml + persistence: + volumes: + - name: custom-attributes + configMap: + name: custom-attributes + #- name: custom-attributes + # secret: + # secretName: custom-attributes + # -- Configure any additional volumesMounts that need to be attached to the containers + volumeMounts: + - mountPath: "/app/custom_ldif/custom_attributes.ldif" + name: custom-attributes + subPath: custom_attributes.ldif + ``` + +4. Run helm upgrade to activate the persistence job. + + ```bash + helm upgrade janssen/janssen -f values.yaml -n + ``` + +Your custom work should be loaded to your persistence. This also persists any changes going forward as you upgrade.