Skip to content

Commit

Permalink
fix: typo and indexing error (#1125)
Browse files Browse the repository at this point in the history 
* fix: rename config httpLoggingExludePaths to httpLoggingExcludePaths

* fix: skip unsupported/invalid indexes

* fix: index creation on spanner persistence
  • Loading branch information
@iromli
iromli authored Mar 29, 2022
1 parent 41b6fa1 commit dc87dc0
Show file tree
Hide file tree
Showing 6 changed files with 21 additions and 75 deletions.
2 changes: 1 addition & 1 deletion docker-jans-auth-server/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ RUN wget -q https://github.com/fabioz/PyDev.Debugger/archive/refs/tags/pydev_deb
# ===========

ENV CN_VERSION=1.0.0-SNAPSHOT
ENV CN_BUILD_DATE='2022-03-23 17:17'
ENV CN_BUILD_DATE='2022-03-28 08:56'
ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war

# Install Jans Auth
Expand Down
2 changes: 1 addition & 1 deletion docker-jans-config-api/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_
# ==========

ENV CN_VERSION=1.0.0-SNAPSHOT
ENV CN_BUILD_DATE='2022-03-23 17:23'
ENV CN_BUILD_DATE='2022-03-28 10:22'
ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war

# Install Jans Config API
Expand Down
2 changes: 1 addition & 1 deletion docker-jans-persistence-loader/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN pip3 install -U pip wheel \
# jans-linux-setup sync
# =====================

ENV JANS_LINUX_SETUP_VERSION=e3d9dbffdab29d58d31dab004f5d392f5ada0591
ENV JANS_LINUX_SETUP_VERSION=41b6fa185505d3a6a1b5423a6f6df38337a14168
ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup

# note that as we're pulling from a monorepo (with multiple project in it)
Expand Down
56 changes: 2 additions & 54 deletions docker-jans-persistence-loader/scripts/spanner_setup.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# import itertools
import hashlib
import json
import logging.config
Expand Down Expand Up @@ -54,11 +53,6 @@ def __init__(self, manager):
with open(f"/app/static/rdbm/{index_fn}") as f:
self.sql_indexes = json.loads(f.read())

# with open("/app/static/couchbase/index.json") as f:
# # prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
# txt = f.read() # .replace("!bucket_prefix!", prefix)
# self.cb_indexes = json.loads(txt)

with open("/app/static/rdbm/sub_tables.json") as f:
self.sub_tables = json.loads(f.read()).get(self.db_dialect) or {}

Expand Down Expand Up @@ -93,31 +87,15 @@ def get_data_type(self, attr, table=None):
syntax_def = self.sql_data_types_mapping[syntax]
type_ = syntax_def.get(self.db_dialect) # or syntax_def["mysql"]

# char_type = "VARCHAR"
# if self.db_dialect == "spanner":
char_type = "STRING"

if type_["type"] != char_type:
# not STRING
data_type = type_["type"]
else:
# if "size" in type_:
# size = type_["size"]
# # data_type = f"{char_type}(type['size'])"
# else:
# # data_type = "STRING(MAX)"
# size = "MAX"
size = type_.get("size") or "MAX"
data_type = f"{char_type}({size})"
# if type_["size"] <= 127:
# data_type = f"{char_type}({type_['size']})"
# elif type_["size"] <= 255:
# data_type = "TINYTEXT" if self.db_dialect == "mysql" else "TEXT"
# else:
# data_type = "TEXT"

# if data_type == "TEXT" and self.db_dialect == "spanner":
# data_type = "STRING(MAX)"

return data_type

def create_tables(self):
Expand Down Expand Up @@ -175,32 +153,9 @@ def create_tables(self):
# sql_cmd = f"ALTER TABLE {table} ADD {col_def};"
# logger.info(sql_cmd)

# def _fields_from_cb_indexes(self):
# fields = []

# for _, data in self.cb_indexes.items():
# # extract and flatten
# attrs = list(itertools.chain.from_iterable(data["attributes"]))
# fields += attrs

# for static in data["static"]:
# attrs = [
# attr for attr in static[0]
# if "(" not in attr
# ]
# fields += attrs

# fields = list(set(fields))
# # exclude objectClass
# if "objectClass" in fields:
# fields.remove("objectClass")
# return fields

def get_index_fields(self, table_name):
# cb_fields = self._fields_from_cb_indexes()
fields = self.sql_indexes.get(table_name, {}).get("fields", [])
fields += self.sql_indexes["__common__"]["fields"]
# fields += cb_fields

# make unique fields
return list(set(fields))
Expand All @@ -214,7 +169,7 @@ def create_spanner_indexes(self, table_name: str, column_mapping: dict):

index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}"

if column_type.lower() != "array":
if not column_type.lower().startswith("array"):
query = f"CREATE INDEX {self.client.quoted_id(index_name)} ON {self.client.quoted_id(table_name)} ({self.client.quoted_id(column_name)})"
self.client.create_index(query)
else:
Expand Down Expand Up @@ -279,12 +234,8 @@ def transform_value(self, key, values):

if data_type in ("DATETIME(3)", "TIMESTAMP",):
dval = values[0].strip("Z")
# sep = " "
# postfix = ""
# if self.db_dialect == "spanner":
sep = "T"
postfix = "Z"
# return "{}-{}-{} {}:{}:{}{}".format(dval[0:4], dval[4:6], dval[6:8], dval[8:10], dval[10:12], dval[12:14], dval[14:17])
return "{}-{}-{}{}{}:{}:{}{}{}".format(
dval[0:4],
dval[4:6],
Expand All @@ -298,7 +249,6 @@ def transform_value(self, key, values):
)

if data_type == "JSON":
# return json.dumps({"v": values})
return {"v": values}

if data_type == "ARRAY<STRING(MAX)>":
Expand All @@ -324,8 +274,6 @@ def data_from_ldif(self, filename):

table_name = oc[-1]

# entry.pop(rdn_name)

if "objectClass" in entry:
entry.pop("objectClass")
elif "objectclass" in entry:
Expand Down
30 changes: 12 additions & 18 deletions docker-jans-persistence-loader/scripts/sql_setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
from pathlib import Path

from ldif import LDIFParser
from sqlalchemy.exc import NotSupportedError
from sqlalchemy.exc import OperationalError

from jans.pycloudlib.persistence.sql import SQLClient

Expand Down Expand Up @@ -91,8 +93,6 @@ def get_data_type(self, attr, table=None):
type_ = syntax_def.get(self.db_dialect) or syntax_def["mysql"]

char_type = "VARCHAR"
if self.db_dialect == "spanner":
char_type = "STRING"

if type_["type"] != char_type:
data_type = type_["type"]
Expand All @@ -104,8 +104,6 @@ def get_data_type(self, attr, table=None):
else:
data_type = "TEXT"

if data_type == "TEXT" and self.db_dialect == "spanner":
data_type = "STRING(MAX)"
return data_type

def create_tables(self):
Expand Down Expand Up @@ -138,8 +136,8 @@ def create_tables(self):
doc_id_type = self.get_data_type("doc_id", table)
table_columns[table].update({
"doc_id": doc_id_type,
"objectClass": "VARCHAR(48)" if self.db_dialect != "spanner" else "STRING(48)",
"dn": "VARCHAR(128)" if self.db_dialect != "spanner" else "STRING(128)",
"objectClass": "VARCHAR(48)",
"dn": "VARCHAR(128)",
})

# make sure ``oc["may"]`` doesn't have duplicate attribute
Expand Down Expand Up @@ -177,9 +175,8 @@ def create_mysql_indexes(self, table_name: str, column_mapping: dict):
if column_name == "doc_id" or column_name not in fields:
continue

index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}"

if column_type.lower() != "json":
index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}"
query = f"CREATE INDEX {self.client.quoted_id(index_name)} ON {self.client.quoted_id(table_name)} ({self.client.quoted_id(column_name)})"
self.client.create_index(query)
else:
Expand All @@ -193,8 +190,13 @@ def create_mysql_indexes(self, table_name: str, column_mapping: dict):
"field": column_name, "data_type": column_type,
})
name = f"{table_name}_json_{i}"
query = f"CREATE INDEX {self.client.quoted_id(name)} ON {self.client.quoted_id(table_name)} (({index_str_fmt}))"
self.client.create_index(query)
query = f"ALTER TABLE {self.client.quoted_id(table_name)} ADD INDEX {self.client.quoted_id(name)} (({index_str_fmt}))"

try:
self.client.create_index(query)
except (NotSupportedError, OperationalError) as exc:
msg = exc.orig.args[1] if self.db_dialect == "mysql" else exc.orig.pgerror
logger.warning(f"Failed to create index {name} for {table_name}.{column_name} column; reason={msg}")

for i, custom in enumerate(self.sql_indexes.get(table_name, {}).get("custom", []), start=1):
# jansPerson table has unsupported custom index expressions that need to be skipped if mysql < 8.0
Expand Down Expand Up @@ -290,10 +292,6 @@ def transform_value(self, key, values):
dval = values[0].strip("Z")
sep = " "
postfix = ""
if self.db_dialect == "spanner":
sep = "T"
postfix = "Z"
# return "{}-{}-{} {}:{}:{}{}".format(dval[0:4], dval[4:6], dval[6:8], dval[8:10], dval[10:12], dval[12:14], dval[14:17])
return "{}-{}-{}{}{}:{}:{}{}{}".format(
dval[0:4],
dval[4:6],
Expand All @@ -307,12 +305,8 @@ def transform_value(self, key, values):
)

if data_type == "JSON":
# return json.dumps({"v": values})
return {"v": values}

if data_type == "ARRAY<STRING(MAX)>":
return values

# fallback
return values[0]

Expand Down
4 changes: 4 additions & 0 deletions docker-jans-persistence-loader/scripts/upgrade.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ def _transform_auth_dynamic_config(conf):
conf["useHighestLevelScriptIfAcrScriptNotFound"] = True
should_update = True

if "httpLoggingExcludePaths" not in conf:
conf["httpLoggingExcludePaths"] = conf.pop("httpLoggingExludePaths", [])
should_update = True

# return the conf and flag to determine whether it needs update or not
return conf, should_update

Expand Down

0 comments on commit dc87dc0

Please sign in to comment.