fix: typo and indexing error

docker-jans-auth-server/Dockerfile

@@ -53,7 +53,7 @@ RUN wget -q https://github.com/fabioz/PyDev.Debugger/archive/refs/tags/pydev_deb
 # ===========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2022-03-23 17:17'
+ENV CN_BUILD_DATE='2022-03-28 08:56'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war
 
 # Install Jans Auth

docker-jans-config-api/Dockerfile

@@ -33,7 +33,7 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_
 # ==========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2022-03-23 17:23'
+ENV CN_BUILD_DATE='2022-03-28 10:22'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
 
 # Install Jans Config API

docker-jans-persistence-loader/Dockerfile

@@ -22,7 +22,7 @@ RUN pip3 install -U pip wheel \
 # jans-linux-setup sync
 # =====================
 
-ENV JANS_LINUX_SETUP_VERSION=e3d9dbffdab29d58d31dab004f5d392f5ada0591
+ENV JANS_LINUX_SETUP_VERSION=41b6fa185505d3a6a1b5423a6f6df38337a14168
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 
 # note that as we're pulling from a monorepo (with multiple project in it)

docker-jans-persistence-loader/scripts/spanner_setup.py

@@ -1,4 +1,3 @@
-# import itertools
 import hashlib
 import json
 import logging.config
@@ -54,11 +53,6 @@ def __init__(self, manager):
         with open(f"/app/static/rdbm/{index_fn}") as f:
             self.sql_indexes = json.loads(f.read())
 
-        # with open("/app/static/couchbase/index.json") as f:
-        #     # prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
-        #     txt = f.read()  # .replace("!bucket_prefix!", prefix)
-        #     self.cb_indexes = json.loads(txt)
-
         with open("/app/static/rdbm/sub_tables.json") as f:
             self.sub_tables = json.loads(f.read()).get(self.db_dialect) or {}
 
@@ -93,31 +87,15 @@ def get_data_type(self, attr, table=None):
         syntax_def = self.sql_data_types_mapping[syntax]
         type_ = syntax_def.get(self.db_dialect)  # or syntax_def["mysql"]
 
-        # char_type = "VARCHAR"
-        # if self.db_dialect == "spanner":
         char_type = "STRING"
 
         if type_["type"] != char_type:
             # not STRING
             data_type = type_["type"]
         else:
-            # if "size" in type_:
-            #     size = type_["size"]
-            #     # data_type = f"{char_type}(type['size'])"
-            # else:
-            #     # data_type = "STRING(MAX)"
-            #     size = "MAX"
             size = type_.get("size") or "MAX"
             data_type = f"{char_type}({size})"
-            # if type_["size"] <= 127:
-            #     data_type = f"{char_type}({type_['size']})"
-            # elif type_["size"] <= 255:
-            #     data_type = "TINYTEXT" if self.db_dialect == "mysql" else "TEXT"
-            # else:
-            #     data_type = "TEXT"
-
-        # if data_type == "TEXT" and self.db_dialect == "spanner":
-        #     data_type = "STRING(MAX)"
+
         return data_type
 
     def create_tables(self):
@@ -175,32 +153,9 @@ def create_tables(self):
         #     sql_cmd = f"ALTER TABLE {table} ADD {col_def};"
         #     logger.info(sql_cmd)
 
-    # def _fields_from_cb_indexes(self):
-    #     fields = []
-
-    #     for _, data in self.cb_indexes.items():
-    #         # extract and flatten
-    #         attrs = list(itertools.chain.from_iterable(data["attributes"]))
-    #         fields += attrs
-
-    #         for static in data["static"]:
-    #             attrs = [
-    #                 attr for attr in static[0]
-    #                 if "(" not in attr
-    #             ]
-    #             fields += attrs
-
-    #     fields = list(set(fields))
-    #     # exclude objectClass
-    #     if "objectClass" in fields:
-    #         fields.remove("objectClass")
-    #     return fields
-
     def get_index_fields(self, table_name):
-        # cb_fields = self._fields_from_cb_indexes()
         fields = self.sql_indexes.get(table_name, {}).get("fields", [])
         fields += self.sql_indexes["__common__"]["fields"]
-        # fields += cb_fields
 
         # make unique fields
         return list(set(fields))
@@ -214,7 +169,7 @@ def create_spanner_indexes(self, table_name: str, column_mapping: dict):
 
             index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}"
 
-            if column_type.lower() != "array":
+            if not column_type.lower().startswith("array"):
                 query = f"CREATE INDEX {self.client.quoted_id(index_name)} ON {self.client.quoted_id(table_name)} ({self.client.quoted_id(column_name)})"
                 self.client.create_index(query)
             else:
@@ -279,12 +234,8 @@ def transform_value(self, key, values):
 
         if data_type in ("DATETIME(3)", "TIMESTAMP",):
             dval = values[0].strip("Z")
-            # sep = " "
-            # postfix = ""
-            # if self.db_dialect == "spanner":
             sep = "T"
             postfix = "Z"
-            # return "{}-{}-{} {}:{}:{}{}".format(dval[0:4], dval[4:6], dval[6:8], dval[8:10], dval[10:12], dval[12:14], dval[14:17])
             return "{}-{}-{}{}{}:{}:{}{}{}".format(
                 dval[0:4],
                 dval[4:6],
@@ -298,7 +249,6 @@ def transform_value(self, key, values):
             )
 
         if data_type == "JSON":
-            # return json.dumps({"v": values})
             return {"v": values}
 
         if data_type == "ARRAY<STRING(MAX)>":
@@ -324,8 +274,6 @@ def data_from_ldif(self, filename):
 
                 table_name = oc[-1]
 
-                # entry.pop(rdn_name)
-
                 if "objectClass" in entry:
                     entry.pop("objectClass")
                 elif "objectclass" in entry:

docker-jans-persistence-loader/scripts/sql_setup.py

@@ -9,6 +9,8 @@
 from pathlib import Path
 
 from ldif import LDIFParser
+from sqlalchemy.exc import NotSupportedError
+from sqlalchemy.exc import OperationalError
 
 from jans.pycloudlib.persistence.sql import SQLClient
 
@@ -91,8 +93,6 @@ def get_data_type(self, attr, table=None):
         type_ = syntax_def.get(self.db_dialect) or syntax_def["mysql"]
 
         char_type = "VARCHAR"
-        if self.db_dialect == "spanner":
-            char_type = "STRING"
 
         if type_["type"] != char_type:
             data_type = type_["type"]
@@ -104,8 +104,6 @@ def get_data_type(self, attr, table=None):
             else:
                 data_type = "TEXT"
 
-        if data_type == "TEXT" and self.db_dialect == "spanner":
-            data_type = "STRING(MAX)"
         return data_type
 
     def create_tables(self):
@@ -138,8 +136,8 @@ def create_tables(self):
             doc_id_type = self.get_data_type("doc_id", table)
             table_columns[table].update({
                 "doc_id": doc_id_type,
-                "objectClass": "VARCHAR(48)" if self.db_dialect != "spanner" else "STRING(48)",
-                "dn": "VARCHAR(128)" if self.db_dialect != "spanner" else "STRING(128)",
+                "objectClass": "VARCHAR(48)",
+                "dn": "VARCHAR(128)",
             })
 
             # make sure ``oc["may"]`` doesn't have duplicate attribute
@@ -177,9 +175,8 @@ def create_mysql_indexes(self, table_name: str, column_mapping: dict):
             if column_name == "doc_id" or column_name not in fields:
                 continue
 
-            index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}"
-
             if column_type.lower() != "json":
+                index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}"
                 query = f"CREATE INDEX {self.client.quoted_id(index_name)} ON {self.client.quoted_id(table_name)} ({self.client.quoted_id(column_name)})"
                 self.client.create_index(query)
             else:
@@ -193,8 +190,13 @@ def create_mysql_indexes(self, table_name: str, column_mapping: dict):
                         "field": column_name, "data_type": column_type,
                     })
                     name = f"{table_name}_json_{i}"
-                    query = f"CREATE INDEX {self.client.quoted_id(name)} ON {self.client.quoted_id(table_name)} (({index_str_fmt}))"
-                    self.client.create_index(query)
+                    query = f"ALTER TABLE {self.client.quoted_id(table_name)} ADD INDEX {self.client.quoted_id(name)} (({index_str_fmt}))"
+
+                    try:
+                        self.client.create_index(query)
+                    except (NotSupportedError, OperationalError) as exc:
+                        msg = exc.orig.args[1] if self.db_dialect == "mysql" else exc.orig.pgerror
+                        logger.warning(f"Failed to create index {name} for {table_name}.{column_name} column; reason={msg}")
 
         for i, custom in enumerate(self.sql_indexes.get(table_name, {}).get("custom", []), start=1):
             # jansPerson table has unsupported custom index expressions that need to be skipped if mysql < 8.0
@@ -290,10 +292,6 @@ def transform_value(self, key, values):
             dval = values[0].strip("Z")
             sep = " "
             postfix = ""
-            if self.db_dialect == "spanner":
-                sep = "T"
-                postfix = "Z"
-            # return "{}-{}-{} {}:{}:{}{}".format(dval[0:4], dval[4:6], dval[6:8], dval[8:10], dval[10:12], dval[12:14], dval[14:17])
             return "{}-{}-{}{}{}:{}:{}{}{}".format(
                 dval[0:4],
                 dval[4:6],
@@ -307,12 +305,8 @@ def transform_value(self, key, values):
             )
 
         if data_type == "JSON":
-            # return json.dumps({"v": values})
             return {"v": values}
 
-        if data_type == "ARRAY<STRING(MAX)>":
-            return values
-
         # fallback
         return values[0]
 

docker-jans-persistence-loader/scripts/upgrade.py

@@ -115,6 +115,10 @@ def _transform_auth_dynamic_config(conf):
         conf["useHighestLevelScriptIfAcrScriptNotFound"] = True
         should_update = True
 
+    if "httpLoggingExcludePaths" not in conf:
+        conf["httpLoggingExcludePaths"] = conf.pop("httpLoggingExludePaths", [])
+        should_update = True
+
     # return the conf and flag to determine whether it needs update or not
     return conf, should_update