diff --git a/setup.py b/setup.py index 1a5b110..e7e6385 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def run_tests(self): author_email="roland@catalogix.se", license="Apache 2.0", url='https://github.com/IdentityPython/oidcmsg/', - packages=["oidcmsg", "oidcmsg/oauth2", "oidcmsg/oidc", 'oidcmsg/storage'], + packages=["oidcmsg", "oidcmsg/oauth2", "oidcmsg/oidc"], package_dir={"": "src"}, classifiers=[ "Development Status :: 4 - Beta", diff --git a/src/oidcmsg/__init__.py b/src/oidcmsg/__init__.py index a3f9a90..30a5958 100755 --- a/src/oidcmsg/__init__.py +++ b/src/oidcmsg/__init__.py @@ -1,5 +1,5 @@ __author__ = "Roland Hedberg" -__version__ = "1.3.2" +__version__ = "1.3.3" import os from typing import Dict diff --git a/src/oidcmsg/item.py b/src/oidcmsg/item.py index 77e90cd..70e393c 100644 --- a/src/oidcmsg/item.py +++ b/src/oidcmsg/item.py @@ -1,10 +1,11 @@ from typing import List from typing import Optional +from cryptojwt.utils import importer +from cryptojwt.utils import qualified_name + from oidcmsg.impexp import ImpExp from oidcmsg.message import Message -from oidcmsg.storage import importer -from oidcmsg.storage.utils import qualified_name class DLDict(ImpExp): diff --git a/src/oidcmsg/oidc/identity_assurance.py b/src/oidcmsg/oidc/identity_assurance.py index 9a65032..f50ac08 100644 --- a/src/oidcmsg/oidc/identity_assurance.py +++ b/src/oidcmsg/oidc/identity_assurance.py @@ -2,6 +2,8 @@ import datetime import json +from cryptojwt.utils import importer + from oidcmsg.message import OPTIONAL_MESSAGE from oidcmsg.message import SINGLE_OPTIONAL_JSON from oidcmsg.message import SINGLE_OPTIONAL_STRING @@ -17,7 +19,6 @@ from oidcmsg.oidc import claims_request_deser from oidcmsg.oidc import deserialize_from_one_of from oidcmsg.oidc import msg_ser_json -from oidcmsg.storage import importer class PlaceOfBirth(Message): diff --git a/src/oidcmsg/storage/__init__.py b/src/oidcmsg/storage/__init__.py deleted file mode 100644 index fa9daea..0000000 --- a/src/oidcmsg/storage/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -import logging - -from .utils import importer - -logger = logging.getLogger(__name__) - - -class Storage(object): - """ - Offers a standard set of methods and I/O on persistent data. - """ - - def __init__(self, conf_dict=None): - pass - - def get(self, k, default=None): - raise NotImplemented() - - def update(self, ava): - raise NotImplemented() - - def delete(self, k, v): - raise NotImplemented() - - def __getitem__(self, k): - raise NotImplemented() - - def __setitem__(self, k, v): - raise NotImplemented() - - def __delitem__(self, v): - raise NotImplemented() - - def __call__(self): - raise NotImplemented() - - def __len__(self): - raise NotImplemented() - - def __contains__(self, k): - raise NotImplemented() - - def __iter__(self): - raise NotImplemented() - - def synch(self): - raise NotImplemented() - - def keys(self): - raise NotImplemented() diff --git a/src/oidcmsg/storage/abfile.py b/src/oidcmsg/storage/abfile.py deleted file mode 100644 index 0d15e49..0000000 --- a/src/oidcmsg/storage/abfile.py +++ /dev/null @@ -1,362 +0,0 @@ -import logging -import os -import time -from urllib.parse import quote_plus - -from filelock import FileLock - -from . import Storage -from .converter import PassThru -from .converter import QPKey -from .extension import key_label -from .utils import importer - -logger = logging.getLogger(__name__) - - -class AbstractFileSystem(Storage): - """ - FileSystem implements a simple file based database. - It has a dictionary like interface. - Each key maps one-to-one to a file on disc, where the content of the - file is the value. - ONLY goes one level deep. - Not directories in directories. - """ - - def __init__( - self, - conf_dict, - ): - """ - items = FileSystem( - { - 'fdir': fdir, - 'key_conv':{'to': quote_plus, 'from': unquote_plus}, - 'value_conv':{'to': keyjar_to_jwks, 'from': jwks_to_keyjar} - }) - - fdir: The root of the directory - key_conv: Converts to/from the key displayed by this class to - users of it to something that can be used as a file name. - The value of key_conv is a class that has the methods 'serialize'/'deserialize'. - value_conv: As with key_conv you can convert/translate - the value bound to a key in the database to something that can easily - be stored in a file. Like with key_conv the value of this parameter - is a class that has the methods 'serialize'/'deserialize'. - """ - - super().__init__(conf_dict) - self.config = conf_dict - - _fdir = conf_dict.get("fdir", ".") - if "issuer" in conf_dict: - _fdir = os.path.join(_fdir, quote_plus(conf_dict["issuer"])) - - self.fdir = _fdir - self.fmtime = {} - self.storage = {} - - key_conv = conf_dict.get("key_conv") - if key_conv: - self.key_conv = importer(key_conv)() - else: - self.key_conv = QPKey() - - value_conv = conf_dict.get("value_conv") - if value_conv: - self.value_conv = importer(value_conv)() - else: - self.value_conv = PassThru() - - if not os.path.isdir(self.fdir): - os.makedirs(self.fdir) - - self.synch() - - def get(self, item, default=None): - try: - return self[item] - except KeyError: - return default - - def __getitem__(self, item): - """ - Return the value bound to an identifier. - - :param item: The identifier. - :return: - """ - item = self.key_conv.serialize(item) - fname = os.path.join(self.fdir, item) - if self._is_file(fname): - lock = FileLock("{}.lock".format(fname)) - with lock: - if self.is_changed(item, fname): - logger.info("File content change in {}".format(item)) - self.storage[item] = self._read_info(fname) - - logger.debug('Read from "%s"', item) - return self.storage[item] - else: - raise KeyError(item) - - def __setitem__(self, key, value): - """ - Binds a value to a specific key. If the file that the key maps to - does not exist it will be created. The content of the file will be - set to the value given. - - :param key: Identifier - :param value: Value that should be bound to the identifier. - :return: - """ - - if not os.path.isdir(self.fdir): - os.makedirs(self.fdir, exist_ok=True) - - try: - _key = self.key_conv.serialize(key) - except KeyError: - _key = key - - fname = os.path.join(self.fdir, _key) - lock = FileLock("{}.lock".format(fname)) - with lock: - with open(fname, "w") as fp: - fp.write(self.value_conv.serialize(value)) - - self.storage[_key] = value - logger.debug('Wrote to "%s"', key) - self.fmtime[_key] = self.get_mtime(fname) - - def __delitem__(self, key): - fname = os.path.join(self.fdir, key) - if os.path.isfile(fname): - lock = FileLock("{}.lock".format(fname)) - with lock: - os.unlink(fname) - - try: - del self.storage[key] - except KeyError: - pass - - def keys(self): - """ - Implements the dict.keys() method - """ - self.synch() - for k in self.storage.keys(): - yield self.key_conv.deserialize(k) - - @staticmethod - def get_mtime(fname): - """ - Find the time this file was last modified. - - :param fname: File name - :return: The last time the file was modified. - """ - try: - mtime = os.stat(fname).st_mtime_ns - except OSError: - # The file might be right in the middle of being written - # so sleep - time.sleep(1) - mtime = os.stat(fname).st_mtime_ns - - return mtime - - def _is_file(self, fname): - return os.path.isfile(fname) - - def is_changed(self, item, fname): - """ - Find out if this item has been modified since last. - When I get here I know that item points to an existing file. - - :param item: A key - :return: True/False - """ - mtime = self.get_mtime(fname) - - try: - _ftime = self.fmtime[item] - except KeyError: # Never been seen before - self.fmtime[item] = mtime - return True - - if mtime > _ftime: # has changed - self.fmtime[item] = mtime - return True - else: - return False - - def _read_info(self, fname): - if os.path.isfile(fname): - try: - lock = FileLock("{}.lock".format(fname)) - with lock: - info = open(fname, "r").read().strip() - return self.value_conv.deserialize(info) - except Exception as err: - logger.error(err) - raise - else: - logger.error("No such file: {}".format(fname)) - return None - - def synch(self): - """ - Goes through the directory and builds a local cache based on - the content of the directory. - """ - if not os.path.isdir(self.fdir): - os.makedirs(self.fdir) - # raise ValueError('No such directory: {}'.format(self.fdir)) - for f in os.listdir(self.fdir): - fname = os.path.join(self.fdir, f) - - if not os.path.isfile(fname): - continue - if fname.endswith(".lock"): - continue - - if f in self.fmtime: - if self.is_changed(f, fname): - self.storage[f] = self._read_info(fname) - else: - mtime = self.get_mtime(fname) - try: - self.storage[f] = self._read_info(fname) - except Exception as err: - logger.warning("Bad content in {} ({})".format(fname, err)) - else: - self.fmtime[f] = mtime - - def items(self): - """ - Implements the dict.items() method - """ - self.synch() - for k, v in self.storage.items(): - yield self.key_conv.deserialize(k), v - - def clear(self): - """ - Completely resets the database. This means that all information in - the local cache and on disc will be erased. - """ - if not os.path.isdir(self.fdir): - os.makedirs(self.fdir, exist_ok=True) - return - - for f in os.listdir(self.fdir): - del self[f] - - def update(self, ava): - """ - Replaces what's in the database with a set of key, value pairs. - Only data bound to keys that appear in ava will be affected. - - :param ava: Dictionary - """ - for key, val in ava.items(): - self[key] = val - - def __contains__(self, item): - return self.key_conv.serialize(item) in self.storage - - def __iter__(self): - return self.items() - - def __call__(self, *args, **kwargs): - return [self.key_conv.deserialize(k) for k in self.storage.keys()] - - def __len__(self): - if not os.path.isdir(self.fdir): - return 0 - - n = 0 - for f in os.listdir(self.fdir): - fname = os.path.join(self.fdir, f) - - if not os.path.isfile(fname): - continue - if fname.endswith(".lock"): - continue - - n += 1 - return n - - def __str__(self): - return "{config:" + str(self.config) + ", info:" + str(self.storage) + "}" - - -class LabeledAbstractFileSystem(Storage): - def __init__(self, conf_dict, label=""): - _conf = {k: v for k, v in conf_dict.items() if k != "label"} - Storage.__init__(self, conf_dict=_conf) - - self.storage = AbstractFileSystem(conf_dict) - _label = label or conf_dict.get("label", "") - - if not _label: - self.label = "" - else: - self.label = "__{}__".format(_label) - - @key_label - def get(self, k, default=None): - return self.storage.get(k, default) - - @key_label - def update(self, ava): - return self.storage.update(ava) - - @key_label - def __getitem__(self, k): - return self.storage.get(k) - - @key_label - def __setitem__(self, k, v): - self.storage[k] = v - - @key_label - def __delitem__(self, k): - del self.storage[k] - - @key_label - def __contains__(self, k): - return self.storage.__contains__(k) - - def __iter__(self): - for key, val in self.storage.__iter__(): - if key.startswith(self.label): - yield key[len(self.label) :], val - - def keys(self): - return [k[len(self.label) :] for k in self.storage.keys() if k.startswith(self.label)] - - def items(self): - for key, val in self.storage.__iter__(): - if key.startswith(self.label): - yield key[len(self.label) :], val - - def __len__(self): - if not os.path.isdir(self.storage.fdir): - return 0 - - n = 0 - for f in os.listdir(self.storage.fdir): - if f.startswith(self.label): - fname = os.path.join(self.storage.fdir, f) - - if not os.path.isfile(fname): - continue - if fname.endswith(".lock"): - continue - - n += 1 - return n diff --git a/src/oidcmsg/storage/absqlalchemy.py b/src/oidcmsg/storage/absqlalchemy.py deleted file mode 100644 index 96eeb48..0000000 --- a/src/oidcmsg/storage/absqlalchemy.py +++ /dev/null @@ -1,88 +0,0 @@ -import datetime -import json - -import sqlalchemy as alchemy_db -from sqlalchemy.orm import scoped_session -from sqlalchemy.orm import sessionmaker - -PlainDict = dict - - -class AbstractStorageSQLAlchemy: - def __init__(self, conf_dict): - self.engine = alchemy_db.create_engine(conf_dict["url"]) - self.connection = self.engine.connect() - self.metadata = alchemy_db.MetaData() - self.table = alchemy_db.Table( - conf_dict["params"]["table"], self.metadata, autoload=True, autoload_with=self.engine - ) - Session = sessionmaker(bind=self.engine) - self.session = scoped_session(Session) - - def get(self, k): - entry = self.session.query(self.table).filter_by(owner=k).first() - if entry is None: - return None - return entry.data - - def set(self, k, v): - self.delete(k) - ins = self.table.insert().values(owner=k, data=v) - self.session.execute(ins) - self.session.commit() - return 1 - - def update(self, k, v): - """ - k = value_to_match - v = value_to_be_substituted - """ - upquery = self.table.update().where(self.table.c.owner == k).values(**{"data": v}) - self.session.execute(upquery) - self.session.commit() - return 1 - - def delete(self, v): - """ - return the count of deleted objects - """ - delquery = self.table.delete().where(self.table.c.owner == v) - n_entries = self.session.query(self.table).filter(self.table.c.owner == v).count() - self.session.execute(delquery) - return n_entries - - def __contains__(self, k): - for entry in self(): - if k in entry: - return 1 - - def __call__(self): - return self.session.query(self.table).all() - - def __iter__(self): - return self.session.query(self.table) - - def __str__(self): - entries = [] - for entry in self(): - l = [] - for element in entry: - if isinstance(element, datetime.datetime): - l.append(element.isoformat()) - else: - l.append(element) - entries.append(l) - return json.dumps(entries, indent=2) - - def flush(self): - """ - make a decision here ... - """ - try: - self.session.commit() - except: - self.session.rollback() - self.session.flush() - - def __setitem__(self, k, v): - return self.set(k, v) diff --git a/src/oidcmsg/storage/converter.py b/src/oidcmsg/storage/converter.py deleted file mode 100644 index 0c08947..0000000 --- a/src/oidcmsg/storage/converter.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -from urllib.parse import quote_plus -from urllib.parse import unquote_plus - - -class QPKey: - def serialize(self, str): - return quote_plus(str) - - def deserialize(self, str): - return unquote_plus(str) - - -class JSON: - def serialize(self, item): - return json.dumps(item) - - def deserialize(self, str): - return json.loads(str) - - -class PassThru: - def serialize(self, str): - return str - - def deserialize(self, str): - return str diff --git a/src/oidcmsg/storage/db_setup.py b/src/oidcmsg/storage/db_setup.py deleted file mode 100644 index 023bc65..0000000 --- a/src/oidcmsg/storage/db_setup.py +++ /dev/null @@ -1,26 +0,0 @@ -import datetime - -import sqlalchemy as alchemy_db -from sqlalchemy.ext.declarative import declarative_base - -Base = declarative_base() - - -class Thing(Base): - __tablename__ = "thing" - - id = alchemy_db.Column( - alchemy_db.Integer, alchemy_db.Sequence("thing_id_seq"), primary_key=True - ) - owner = alchemy_db.Column(alchemy_db.String(80), unique=False, nullable=False) - data = alchemy_db.Column(alchemy_db.String(4096), unique=False, nullable=False) - created = alchemy_db.Column(alchemy_db.DateTime, default=datetime.datetime.utcnow) - - def __repr__(self): - return "" % self.owner - - -def create_database(conf_dict): - engine = alchemy_db.create_engine(conf_dict["url"]) - connection = engine.connect() - Base.metadata.create_all(engine) diff --git a/src/oidcmsg/storage/extension.py b/src/oidcmsg/storage/extension.py deleted file mode 100644 index d6f3d4a..0000000 --- a/src/oidcmsg/storage/extension.py +++ /dev/null @@ -1,68 +0,0 @@ -def key_label(func): - def add_label(self, k, *args): - _k = "{}{}".format(self.label, k) - return func(self, _k, *args) - - return add_label - - -class SetGetDict(dict): - def set(self, key, value): - self[key] = value - - def get(self, item, default=None): - try: - return self[item] - except KeyError: - return default - - def delete(self, item): - del self[item] - - -class LabeledDict: - def __init__(self, label=""): - self.storage = SetGetDict() - - if label == "": - self.label = label - self.label_len = 0 - else: - self.label = "__{}__".format(label) - self.label_len = len(self.label) - - @key_label - def get(self, k, default=None): - return self.storage.get(k, default) - - @key_label - def update(self, ava): - return self.storage.update(ava) - - @key_label - def __getitem__(self, k): - return self.storage.get(k) - - @key_label - def __setitem__(self, k, v): - return self.storage.set(k, v) - - @key_label - def set(self, k, v): - return self.storage.set(k, v) - - @key_label - def __delitem__(self, k): - del self.storage[k] - - @key_label - def __contains__(self, k): - return self.storage.__contains__(k) - - def __iter__(self): - for key, val in self.storage.__iter__(): - if key.startswith(self.label): - yield key[self.label_len :], val - - def keys(self): - return [k[self.label_len :] for k in self.storage.keys() if k.startswith(self.label)] diff --git a/src/oidcmsg/storage/init.py b/src/oidcmsg/storage/init.py deleted file mode 100644 index 05bdaea..0000000 --- a/src/oidcmsg/storage/init.py +++ /dev/null @@ -1,65 +0,0 @@ -from .extension import LabeledDict -from .utils import importer - -""" -Configuration example - -STORAGE_CONFIG_1: { - 'keyjar': { - 'handler': 'abstorage.storages.abfile.AbstractFileSystem', - 'fdir': 'db/keyjar', - 'key_conv': 'abstorage.converter.QPKey', - 'value_conv': 'cryptojwt.serialize.item.KeyIssuer', - }, - 'default': { - 'handler': 'abstorage.storages.abfile.AbstractFileSystem', - 'fdir': 'db', - 'key_conv': 'abstorage.converter.QPKey', - 'value_conv': 'abstorage.converter.JSON' - } -} -""" - - -class ConfigurationError(Exception): - pass - - -def get_storage_conf(db_conf=None, typ="default"): - _conf = None - if db_conf: - _conf = db_conf.get(typ) - if _conf: - return _conf - elif typ != "default": - _conf = db_conf.get("default") - else: - raise ConfigurationError() - - return _conf - - -def storage_factory(configuration): - _handler = configuration.get("handler") - if _handler: - storage_cls = importer(_handler) - else: - raise ConfigurationError("Missing handler specification") - _conf = {k: v for k, v in configuration.items() if k != "handler"} - return storage_cls(_conf) - - -def init_storage(db_conf=None, key="default"): - """ - Returns a storage instance. - - :param conf: Configuration - :param key: Which part of the configuration is to be used - """ - - if db_conf: - _conf = get_storage_conf(db_conf, key) - if _conf: - return storage_factory(_conf) - - return LabeledDict({"label": key}) diff --git a/src/oidcmsg/storage/utils.py b/src/oidcmsg/storage/utils.py deleted file mode 100644 index d7525c3..0000000 --- a/src/oidcmsg/storage/utils.py +++ /dev/null @@ -1,28 +0,0 @@ -import importlib - - -def modsplit(name): - """Split importable""" - if ":" in name: - _part = name.split(":") - if len(_part) != 2: - raise ValueError("Syntax error: {s}") - return _part[0], _part[1] - - _part = name.split(".") - if len(_part) < 2: - raise ValueError("Syntax error: {s}") - - return ".".join(_part[:-1]), _part[-1] - - -def importer(name): - """Import by name""" - _part = modsplit(name) - module = importlib.import_module(_part[0]) - return getattr(module, _part[1]) - - -def qualified_name(cls): - """ Go from class instance to name usable for imports. """ - return cls.__module__ + "." + cls.__name__