pi-manage

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# 2018-08-07 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#            Allow creation of HSM keys
# 2017-10-08 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#            Allow cleaning up different actions with different
#            retention times.
# 2017-07-12 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#            Add generation of PGP keys
# 2017-02-23 Cornelius Kölbel <cornelius.koelbel@netknights.it>
#            Add CA sub commands
# 2017-01-27 Diogenes S. Jesus
#            Cornelius Kölbel <cornelius.koelbel@netknights.it>
#            Add creation of more detailed policy
# 2016-04-15 Cornelius Kölbel <cornelius@privacyidea.org>
#            Add backup for pymysql driver
# 2016-01-29 Cornelius Kölbel <cornelius@privacyidea.org>
#            Add profiling
# 2015-10-09 Cornelius Kölbel <cornelius@privacyidea.org>
#            Set file permissions
# 2015-09-24 Cornelius Kölbel <cornelius@privacyidea.org>
#            Add validate call
# 2015-06-16 Cornelius Kölbel <cornelius@privacyidea.org>
#            Add creation of JWT token
# 2015-03-27 Cornelius Kölbel, cornelius@privacyidea.org
#            Add sub command for policies
# 2014-12-15 Cornelius Kölbel, info@privacyidea.org
#            Initial creation
#
# (c) Cornelius Kölbel
# Info: http://www.privacyidea.org
#
# This code is free software; you can redistribute it and/or
# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
# License as published by the Free Software Foundation; either
# version 3 of the License, or any later version.
#
# This code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# ./manage.py db init
# ./manage.py db migrate
# ./manage.py createdb
#
from __future__ import print_function
import os
import sys
import datetime
from datetime import timedelta
import re
from subprocess import call, Popen, PIPE
from getpass import getpass
import gnupg
import yaml
import contextlib
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
import flask
from six.moves import shlex_quote

from privacyidea.lib.sqlutils import delete_matching_rows
from privacyidea.lib.security.default import DefaultSecurityModule
from privacyidea.lib.crypto import geturandom
from privacyidea.lib.auth import (create_db_admin, list_db_admin,
                                  delete_db_admin)
from privacyidea.lib.policy import (delete_policy, enable_policy,
                                    PolicyClass, set_policy)
from privacyidea.lib.event import (delete_event, enable_event,
                                   set_event, EventConfiguration)
from privacyidea.lib.caconnector import (get_caconnector_list,
                                         get_caconnector_class,
                                         get_caconnector_object,
                                         save_caconnector)
from privacyidea.app import create_app
from privacyidea.lib.auth import ROLE
from flask_script import Manager, Command, Option
from privacyidea.app import db
from flask_migrate import MigrateCommand
# Wee need to import something, so that the models will be created.
from privacyidea.models import Audit
from sqlalchemy import create_engine, desc, MetaData
from sqlalchemy.orm import sessionmaker
from privacyidea.lib.auditmodules.sqlaudit import LogEntry
from privacyidea.lib.audit import getAudit
from privacyidea.lib.authcache import cleanup as authcache_cleanup
from privacyidea.lib.utils import parse_timedelta
from privacyidea.lib.crypto import create_hsm_object
from privacyidea.lib.importotp import parseOATHcsv
from privacyidea.lib.token import import_token
import jwt
import ast
import base64

SILENT = True
MYSQL_DIALECTS = ["mysql", "pymysql", "mysql+pymysql"]

app = create_app(config_name='production', silent=SILENT)
manager = Manager(app)
admin_manager = Manager(usage='Create new administrators or modify existing '
                              'ones.')
backup_manager = Manager(usage='Create database backup and restore')
realm_manager = Manager(usage='Create new realms, delete existing realms '
                              'or set the default realm')
resolver_manager = Manager(usage='Create new resolver')
policy_manager = Manager(usage='Manage policies')
event_manager = Manager(usage='Manage events')
api_manager = Manager(usage="Manage API keys")
ca_manager = Manager(usage="Manage Certificate Authorities")
audit_manager = Manager(usage="Manage Audit log")
authcache_manager = Manager(usage="Manage AuthCache")
hsm_manager = Manager(usage="Manage HSM")
token_manager = Manager(usage="Manage tokens")
manager.add_command('db', MigrateCommand)
manager.add_command('admin', admin_manager)
manager.add_command('backup', backup_manager)
manager.add_command('realm', realm_manager)
manager.add_command('resolver', resolver_manager)
manager.add_command('policy', policy_manager)
manager.add_command('event', event_manager)
manager.add_command('api', api_manager)
manager.add_command('ca', ca_manager)
manager.add_command('audit', audit_manager)
manager.add_command('authcache', authcache_manager)
manager.add_command('hsm', hsm_manager)
manager.add_command('token', token_manager)


@hsm_manager.command
def create_keys():
    """
    Create new encryption keys on the HSM. Be sure to first setup the HSM module, the PKCS11
    module and the slot/password for the given HSM in your pi.cfg.
    Set the variables PI_HSM_MODULE, PI_HSM_MODULE_MODULE, PI_HSM_MODULE_SLOT,
                      PI_HSM_MODULE_PASSWORD.
    """
    hsm_object = create_hsm_object(app.config)
    r = hsm_object.create_keys()
    print("Please add the following to your pi.cfg:")
    print("PI_HSM_MODULE_KEY_LABEL_TOKEN = '{0}'".format(r.get("token")))
    print("PI_HSM_MODULE_KEY_LABEL_CONFIG = '{0}'".format(r.get("config")))
    print("PI_HSM_MODULE_KEY_LABEL_VALUE = '{0}'".format(r.get("value")))


def list_ca(verbose=False):
    """
    List the Certificate Authorities.
    """
    lst = get_caconnector_list()
    for ca in lst:
        print("{ca!s} (type {typ!s})".format(ca=ca.get("connectorname"),
                                             typ=ca.get("type")))
        if verbose:
            for (k, v) in ca.get("data").items():
                print("\t{key!s:20}: {value!s}".format(key=k, value=v))


ca_manager.add_command('list', Command(list_ca))


@ca_manager.command
def create_crl(ca, force=False):
    ca_obj = get_caconnector_object(ca)
    r = ca_obj.create_crl(check_validity=not force)
    if not r:
        print("The CRL was not created.")
    else:
        print("The CRL {name!s} was created.".format(name=r))


@ca_manager.option('name', help='The name of the new CA')
@ca_manager.option('-t', '--type',
                   help='The type of the new CA. The default is "local"',
                   dest='catype')
def create(name, catype='local'):
    """
    Create a new CA connector. In case of the "localca" also the directory
    structure, the openssl.cnf and the key pair is created.
    """
    ca = get_caconnector_object(name)
    if ca:
        print("A CA connector with the name '{0!s}' already exists.".format(
            name))
        sys.exit(1)
    if not catype:
        catype = "local"
    print("Creating CA connector of type {0!s}.".format(catype))
    ca_class = get_caconnector_class(catype)
    ca_params = ca_class.create_ca(name)
    r = save_caconnector(ca_params)
    if r:
        print("Saved CA Connector with ID {0!s}.".format(r))
    else:
        print("Error saving CA connector.")


@admin_manager.command
def add(username, email=None, password=None):
    """
    Register a new administrator in the database.
    """
    db.create_all()
    if not password:
        password = getpass()
        password2 = getpass(prompt='Confirm: ')
        if password != password2:
            import sys
            sys.exit('Error: passwords do not match.')

    create_db_admin(app, username, email, password)
    print('Admin {0} was registered successfully.'.format(username))


def list_admins():
    """
    List all administrators.
    """
    list_db_admin()


admin_manager.add_command('list', Command(list_admins))


@admin_manager.command
def delete(username):
    """
    Delete an existing administrator.
    """
    delete_db_admin(username)


@admin_manager.command
def change(username, email=None, password_prompt=False):
    """
    Change the email address or the password of an existing administrator.
    """
    if password_prompt:
        password = getpass()
        password2 = getpass(prompt='Confirm: ')
        if password != password2:
            import sys
            sys.exit('Error: passwords do not match.')
    else:
        password = None

    create_db_admin(app, username, email, password)


@backup_manager.command
def create(directory="/var/lib/privacyidea/backup/",
           conf_dir="/etc/privacyidea/",
           radius_directory=None,
           enckey=False):
    """
    Create a new backup of the database and the configuration. The default
    does not include the encryption key. Use the 'enckey' option to also
    backup the encryption key. Then you should make sure, that the backups
    are stored safely.

    If you want to also include the RADIUS configuration into the backup
    specify a directory using 'radius_directory'.
    """
    CONF_DIR = conf_dir
    DATE = datetime.datetime.now().strftime("%Y%m%d-%H%M")
    BASE_NAME = "privacyidea-backup"

    directory = os.path.abspath(directory)
    call(["mkdir", "-p", directory])
    # set correct owner, if possible
    if os.geteuid() == 0:
        encfile_stat = os.stat(app.config.get("PI_ENCFILE"))
        os.chown(directory, encfile_stat.st_uid, encfile_stat.st_gid)

    sqlfile = "%s/dbdump-%s.sql" % (directory, DATE)
    backup_file = "%s/%s-%s.tgz" % (directory, BASE_NAME, DATE)

    sqluri = app.config.get("SQLALCHEMY_DATABASE_URI")
    sqltype = sqluri.split(":")[0]
    if sqltype == "sqlite":
        productive_file = sqluri[len("sqlite:///"):]
        print("Backup SQLite %s" % productive_file)
        sqlfile = "%s/db-%s.sqlite" % (directory, DATE)
        call(["cp", productive_file, sqlfile])
    elif sqltype in MYSQL_DIALECTS:
        m = re.match(r".*mysql://(.*):(.*)@(.*)/(\w*)\??(.*)", sqluri)
        username = m.groups()[0]
        password = m.groups()[1]
        datahost = m.groups()[2]
        database = m.groups()[3]
        # We strip parameters, but we do not use them
        _parameters = m.groups()[4]
        defaults_file = "{0!s}/mysql.cnf".format(conf_dir)
        _write_mysql_defaults(defaults_file, username, password)
        call("mysqldump --defaults-file=%s -h %s %s > %s" % (
            shlex_quote(defaults_file),
            shlex_quote(datahost),
            shlex_quote(database),
            shlex_quote(sqlfile)), shell=True)
    else:
        print("unsupported SQL syntax: %s" % sqltype)
        sys.exit(2)
    enc_file = app.config.get("PI_ENCFILE")

    backup_call = ["tar", "-zcf",
                   backup_file, CONF_DIR, sqlfile]

    if radius_directory:
        # Simply append the radius directory to the backup command
        backup_call.append(radius_directory)

    if not enckey:
        # Exclude enckey from backup
        backup_call.append("--exclude={0!s}".format(enc_file))

    call(backup_call)
    os.unlink(sqlfile)
    os.chmod(backup_file, 0o600)


def _write_mysql_defaults(filename, username, password):
    """
    Write the defaults_file for mysql commands

    :param filename: THe name of the file
    :param username: The username to connect to the database
    :param password: The password to connect to the database
    :return:
    """
    with open(filename, "w") as f:
        f.write("""[client]
no-tablespaces=True
user={0!s}
password={1!s}""".format(username, password))

    os.chmod(filename, 0o600)
    # set correct owner, if possible
    if os.geteuid() == 0:
        directory_stat = os.stat(os.path.dirname(filename))
        os.chown(filename, directory_stat.st_uid, directory_stat.st_gid)


@backup_manager.command
def restore(backup_file):
    """
    Restore a previously made backup. You need to specify the tgz file.
    """
    sqluri = None
    config_file = None
    sqlfile = None
    enckey_contained = False

    p = Popen(["tar", "-ztf", backup_file], stdout=PIPE, universal_newlines=True)
    std_out, err_out = p.communicate()
    for line in std_out.split("\n"):
        if re.search(r"/pi.cfg$", line):
            config_file = "/{0!s}".format(line.strip())
        elif re.search(r"\.sql", line):
            sqlfile = "/{0!s}".format(line.strip())
        elif re.search(r"/enckey", line):
            enckey_contained = True

    if not config_file:
        raise Exception("Missing config file pi.cfg in backup file.")
    if not sqlfile:
        raise Exception("Missing database dump in backup file.")

    if enckey_contained:
        print("Also restoring encryption key 'enckey'")
    else:
        print("NO FILE 'enckey' CONTAINED! BE SURE TO RESTORE THE ENCRYPTION "
              "KEY MANUALLY!")
    print("Restoring to {0!s} with data from {1!s}".format(config_file,
                                                           sqlfile))

    call(["tar", "-zxf", backup_file, "-C", "/"])
    print(60*"=")
    with open(config_file, "r") as f:
        # Determine the SQLAlchemy URI
        for line in f:
            if re.search("^SQLALCHEMY_DATABASE_URI", line):
                key, value = line.split("=", 1)
                # Strip whitespaces, and ' "
                sqluri = value.strip().strip("'").strip('"')

    if sqluri is None:
        print("No SQLALCHEMY_DATABASE_URI found in {0!s}".format(config_file))
        sys.exit(2)
    sqltype = sqluri.split(":")[0]
    if sqltype == "sqlite":
        productive_file = sqluri[len("sqlite:///"):]
        print("Restore SQLite %s" % productive_file)
        call(["cp", sqlfile, productive_file])
        os.unlink(sqlfile)
    elif sqltype in MYSQL_DIALECTS:
        m = re.match(r".*mysql://(.*):(.*)@(.*)/(\w*)\??(.*)", sqluri)
        username = m.groups()[0]
        password = m.groups()[1]
        datahost = m.groups()[2]
        database = m.groups()[3]
        defaults_file = "/etc/privacyidea/mysql.cnf"
        _write_mysql_defaults(defaults_file, username, password)
        # Rewriting database
        print("Restoring database.")
        call("mysql --defaults-file=%s -h %s %s < %s" % (shlex_quote(defaults_file),
                                                         shlex_quote(datahost),
                                                         shlex_quote(database),
                                                         shlex_quote(sqlfile)), shell=True)
        os.unlink(sqlfile)
    else:
        print("unsupported SQL syntax: %s" % sqltype)
        sys.exit(2)


@manager.command
def test():
    """
    Run all nosetests.
    """
    call(['nosetests', '-v',
          '--with-coverage', '--cover-package=privacyidea', '--cover-branches',
          '--cover-erase', '--cover-html', '--cover-html-dir=cover'])


@manager.command
def encrypt_enckey(encfile):
    """
    You will be asked for a password and the encryption key in the specified
    file will be encrypted with an AES key derived from your password.

    The encryption key in the file is a 96 bit binary key.

    The password based encrypted encryption key is a hex combination of an IV
    and the encrypted data.

    The result can be piped to a new enckey file.
    """
    # TODO we just print out a string here and assume, the user pipes it into a file.
    #      Maybe we should write the file here so we know what is in there
    password = getpass()
    password2 = getpass(prompt='Confirm: ')
    if password != password2:
        import sys
        sys.exit('Error: passwords do not match.')
    with open(encfile, "rb") as f:
        enckey = f.read()
    res = DefaultSecurityModule.password_encrypt(enckey, password)
    print(res)


@manager.command
def create_enckey(enckey_b64=None):
    """
    If the key of the given configuration does not exist, it will be created.


    :param enckey_b64: (Optional) base64 encoded plain text key
    :return:
    """
    print()
    filename = app.config.get("PI_ENCFILE")
    if os.path.isfile(filename):
        print("The file \n\t%s\nalready exist. We do not overwrite it!" %
              filename)
        sys.exit(1)
    with open(filename, "wb") as f:
        if enckey_b64 is None:
            f.write(DefaultSecurityModule.random(96))
        else:
            print("Warning: Passing enckey via cli input is considered harmful.")
            bin_enckey = base64.b64decode(enckey_b64)
            if len(bin_enckey) != 96:
                print("Error: enckey must be 96 bytes length")
                sys.exit(1)
            f.write(bin_enckey)
    print("Encryption key written to %s" % filename)
    os.chmod(filename, 0o400)
    print("The file permission of %s was set to 400!" % filename)
    print("Please ensure, that it is owned by the right user.   ")


@manager.command
def create_pgp_keys(keysize=2048, force=False):
    """
    Generate PGP keys to allow encrypted token import.
    """
    GPG_HOME = app.config.get("PI_GNUPG_HOME", "/etc/privacyidea/gpg")
    gpg = gnupg.GPG(gnupghome=GPG_HOME)
    keys = gpg.list_keys(True)
    if len(keys) and not force:
        print("There are already private keys. If you want to "
              "generate a new private key, use the parameter --force.")
        print(keys)
        sys.exit(1)
    input_data = gpg.gen_key_input(key_type="RSA", key_length=keysize,
                                   name_real="privacyIDEA Server",
                                   name_comment="Import")
    inputs = input_data.split("\n")
    if inputs[-2] == "%commit":
        del(inputs[-1])
        del(inputs[-1])
        inputs.append("%no-protection")
        inputs.append("%commit")
        inputs.append("")
        input_data = "\n".join(inputs)
    gpg.gen_key(input_data)


@manager.command
def create_audit_keys(keysize=2048):
    """
    Create the RSA signing keys for the audit log.
    You may specify an additional keysize.
    The default keysize is 2048 bit.
    """
    filename = app.config.get("PI_AUDIT_KEY_PRIVATE")
    if os.path.isfile(filename):
        print("The file \n\t%s\nalready exist. We do not overwrite it!" %
              filename)
        sys.exit(1)
    new_key = rsa.generate_private_key(public_exponent=65537,
                                       key_size=keysize,
                                       backend=default_backend())
    priv_pem = new_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.TraditionalOpenSSL,
        encryption_algorithm=serialization.NoEncryption())
    with open(filename, "wb") as f:
        f.write(priv_pem)

    pub_key = new_key.public_key()
    pub_pem = pub_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo)
    with open(app.config.get("PI_AUDIT_KEY_PUBLIC"), "wb") as f:
        f.write(pub_pem)

    print("Signing keys written to %s and %s" %
          (filename, app.config.get("PI_AUDIT_KEY_PUBLIC")))
    os.chmod(filename, 0o400)
    print("The file permission of %s was set to 400!" % filename)
    print("Please ensure, that it is owned by the right user.")


@manager.command
def createdb():
    """
    Initially create the tables in the database. The database must exist.
    (SQLite database will be created)
    """
    print(db)
    db.create_all()
    db.session.commit()


@manager.command
def dropdb(dropit=None):
    """
    This drops all the privacyIDEA database tables (except audit table).
    Use with caution! All data will be lost!

    For safety reason you need to pass
        --dropit==yes
    Otherwise the command will not drop anything.
    """
    if dropit == "yes":
        print("Dropping all database tables!")
        db.drop_all()
    else:
        print("Not dropping anything!")


@manager.command
def validate(user, password, realm=None):
    """
    Do an authentication request
    """
    from privacyidea.lib.user import get_user_from_param
    from privacyidea.lib.token import check_user_pass
    try:
        user = get_user_from_param({"user": user, "realm": realm})
        auth, details = check_user_pass(user, password)
        print("RESULT=%s" % auth)
        print("DETAILS=%s" % details)
    except Exception as exx:
        print("RESULT=Error")
        print("ERROR=%s" % exx)


class CommandOutsideRequestContext(Command):
    """
    In contrast to flask_script's `Command`, this command class does
    not push a request context before running the command.
    """
    def __call__(self, app=None, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *arg, **kwargs):
        pass


def profile(length=30, profile_dir=None):
    """
    Start the application in profiling mode.
    """
    from werkzeug.contrib.profiler import ProfilerMiddleware
    if flask.has_request_context():
        print("WARNING: The app may behave unrealistically during profiling.")
    app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[length],
                                      profile_dir=profile_dir)
    app.run()


# If flask_script's `Command` is used here instead of `CommandOutsideRequestContext`,
# the request context will persist over requests. Thus, `g` is not
# cleared properly between requests, which makes privacyIDEA behave unrealistically.
try:
    manager.add_command('profile', CommandOutsideRequestContext(profile))
except TypeError:
    # Apparently, we are using Flask-Script 0.6.7, which does not support the API used above.
    # So we just add the `profile` command without using `CommandOutsideRequestContext`.
    profile = manager.command(profile)


@authcache_manager.command
def cleanup(minutes=480):
    """
    Remove entries from the authcache, where last_auth entry is older than
    the given number of minutes.
    """
    r = authcache_cleanup(int(minutes))
    print(u"Entries deleted: {0!s}".format(r))


@manager.option('--highwatermark', '--hw', help="If entries exceed this value, "
                                                "old entries are deleted.")
@manager.option('--lowwatermark', '--lw', help="Keep this number of entries.")
@manager.option('--age', help="Delete audit entries older than these number "
                              "of days.")
@manager.option('--config', help="Read config from the specified yaml file.")
@manager.option('--dryrun', help="Do not actually delete, only show "
                                 "what would be done.", action="store_true")
@manager.option('--chunksize', help="Delete entries in chunks of the given size "
                                    "to avoid deadlocks")
@audit_manager.option('--highwatermark', '--hw', help="If entries exceed this value, "
                                                      "old entries are deleted.")
@audit_manager.option('--lowwatermark', '--lw', help="Keep this number of entries.")
@audit_manager.option('--age', help="Delete audit entries older than these number "
                                    "of days.")
@audit_manager.option('--config', help="Read config from the specified yaml file.")
@audit_manager.option('--dryrun', help="Do not actually delete, only show "
                                       "what would be done.", action="store_true")
@audit_manager.option('--chunksize', help="Delete entries in chunks of the given size "
                                          "to avoid deadlocks")
def rotate_audit(highwatermark=10000, lowwatermark=5000, age=0, config=None,
                 dryrun=False, chunksize=None):
    """
    Clean the SQL audit log.

    You can either clean the audit log based on the number of entries of
    based on the age of the entries.

    Cleaning based on number of entries:

    If more than 'highwatermark' entries are in the audit log old entries
    will be deleted, so that 'lowwatermark' entries remain.

    Cleaning based on age:

    Entries older than the specified number of days are deleted.

    Cleaning based on config file:

    You can clean different type of entries with different ages or watermark.
    See the documentation for the format of the config file
    """
    metadata = MetaData()
    highwatermark = int(highwatermark or 10000)
    lowwatermark = int(lowwatermark or 5000)
    if chunksize is not None:
        chunksize = int(chunksize)

    default_module = "privacyidea.lib.auditmodules.sqlaudit"
    token_db_uri = app.config.get("SQLALCHEMY_DATABASE_URI")
    audit_db_uri = app.config.get("PI_AUDIT_SQL_URI", token_db_uri)
    audit_module = app.config.get("PI_AUDIT_MODULE", default_module)
    if audit_module != default_module:
        raise Exception("We only rotate SQL audit module. You are using %s" %
                        audit_module)
    if config:
        print("Cleaning up with config file.")
    elif age:
        age = int(age)
        print("Cleaning up with age: %s. %s" % (age, audit_db_uri))
    else:
        print("Cleaning up with high: %s, low: %s. %s" % (highwatermark,
                                                          lowwatermark,
                                                          audit_db_uri))

    engine = create_engine(audit_db_uri)
    # create a configured "Session" class
    session = sessionmaker(bind=engine)()
    # create a Session
    metadata.create_all(engine)
    if config:
        with open(config, 'r') as f:
            yml_config = yaml.load(f)
        auditlogs = session.query(LogEntry).all()
        delete_list = []
        for log in auditlogs:
            print("investigating log entry {0!s}".format(log.id))
            for rule in yml_config:
                age = int(rule.get("rotate"))
                rotate_date = datetime.datetime.now() - datetime.timedelta(days=age)

                match = False
                for key in rule.keys():
                    if key not in ["rotate"]:
                        search_value = rule.get(key)
                        print(" + searching for {0!r} in {1!s}".format(search_value,
                                                                       getattr(LogEntry, key)))
                        audit_value = getattr(log, key) or ""
                        m = re.search(search_value, audit_value)
                        if m:
                            # it matches!
                            print(" + -- found {0!r}".format(audit_value))
                            match = True
                        else:
                            # It does not match, we continue to next rule
                            print(" + NO MATCH - SKIPPING rest of conditions!")
                            match = False
                            break

                if match:
                    if log.date < rotate_date:
                        # Delete it!
                        print(" + Deleting {0!s} due to rule {1!s}".format(log.id, rule))
                        # Delete it
                        delete_list.append(log.id)
                    # skip all other rules and go to the next log entry
                    break
        if dryrun:
            print("If you only would let me I would clean up "
                  "{0!s} entries!".format(len(delete_list)))
        else:
            print("Cleaning up {0!s} entries.".format(len(delete_list)))
            delete_matching_rows(session, LogEntry.__table__,
                                 LogEntry.id.in_(delete_list), chunksize)
    elif age:
        now = datetime.datetime.now() - datetime.timedelta(days=age)
        print("Deleting entries older than {0!s}".format(now))
        criterion = LogEntry.date < now
        if dryrun:
            r = LogEntry.query.filter(criterion).count()
            print("Would delete {0!s} entries.".format(r))
        else:
            r = delete_matching_rows(session, LogEntry.__table__, criterion, chunksize)
            print("{0!s} entries deleted.".format(r))
    else:
        count = session.query(LogEntry.id).count()
        last_id = 0
        for l in session.query(LogEntry.id).order_by(desc(LogEntry.id)).limit(1):
            last_id = l[0]
        print("The log audit log has %i entries, the last one is %i" % (count,
                                                                        last_id))
        # deleting old entries
        if count > highwatermark:
            print("More than %i entries, deleting..." % highwatermark)
            cut_id = last_id - lowwatermark
            # delete all entries less than cut_id
            print("Deleting entries smaller than %i" % cut_id)
            criterion = LogEntry.id < cut_id
            if dryrun:
                r = LogEntry.query.filter(criterion).count()
            else:
                r = delete_matching_rows(session, LogEntry.__table__, criterion, chunksize)
            print("{0!s} entries deleted.".format(r))


@contextlib.contextmanager
def smartopen(filename):
    if filename and filename != '-':
        fh = open(filename, 'w')
    else:
        fh = sys.stdout

    try:
        yield fh
    finally:
        if fh is not sys.stdout:
            fh.close()


@audit_manager.option('--timelimit', '-t', help="Limit the dumped audit entries to a certain "
                                                "period (i.e. '5d' or '3h' for the entries from "
                                                "the last five days or three hours. By default "
                                                "all audit entries will be dumped.")
@audit_manager.option('--filename', '-f', help="Name of the 'csv' file to dump the audit entries "
                                               "into. By default write to stdout.", default='-')
def dump(filename, timelimit=None):
    """Dump the audit log in csv format."""
    audit = getAudit(app.config)
    tl = parse_timedelta(timelimit) if timelimit else None
    with smartopen(filename) as fh:
        for line in audit.csv_generator(timelimit=tl):
            fh.write(line)


@resolver_manager.command
def create(name, rtype, filename):
    """
    Create a new resolver with name and type (ldapresolver, sqlresolver).
    Read the necessary resolver parameters from the filename. The file should
    contain a python dictionary.

    :param name: The name of the resolver
    :param rtype: The type of the resolver like ldapresolver or sqlresolver
    :param filename: The name of the config file.
    :return:
    """
    from privacyidea.lib.resolver import save_resolver

    with open(filename, 'r') as f:
        contents = f.read()

    params = ast.literal_eval(contents)
    params["resolver"] = name
    params["type"] = rtype
    save_resolver(params)


@resolver_manager.command
def create_internal(name):
    """
    This creates a new internal, editable sqlresolver. The users will be
    stored in the token database in a table called 'users_<name>'. You can then
    add this resolver to a new real using the command 'pi-manage.py realm'.
    """
    from privacyidea.lib.resolver import save_resolver
    sqluri = app.config.get("SQLALCHEMY_DATABASE_URI")
    sqlelements = sqluri.split("/")
    # mysql://user:password@localhost/pi
    # sqlite:////home/cornelius/src/privacyidea/data.sqlite
    sql_driver = sqlelements[0][:-1]
    user_pw_host = sqlelements[2]
    database = "/".join(sqlelements[3:])
    username = ""
    password = ""
    host = ""
    # determine host and user
    hostparts = user_pw_host.split("@")
    if len(hostparts) > 2:
        print("Invalid database URI: %s" % sqluri)
        sys.exit(2)
    elif len(hostparts) == 1:
        host = hostparts[0] or "/"
    elif len(hostparts) == 2:
        host = hostparts[1] or "/"
        # split hostname and password
        userparts = hostparts[0].split(":")
        if len(userparts) == 2:
            username = userparts[0]
            password = userparts[1]
        elif len(userparts) == 1:
            username = userparts[0]
        else:
            print("Invalid username and password in database URI: %s" % sqluri)
            sys.exit(3)
    # now we can create the resolver
    params = {'resolver': name,
              'type': "sqlresolver",
              'Server': host,
              'Driver': sql_driver,
              'User': username,
              'Password': password,
              'Database': database,
              'Table': 'users_' + name,
              'Limit': '500',
              'Editable': '1',
              'Map': '{"userid": "id", "username": "username", '
                     '"email":"email", "password": "password", '
                     '"phone":"phone", "mobile":"mobile", "surname":"surname", '
                     '"givenname":"givenname", "description": "description"}'}
    save_resolver(params)

    # Now we create the database table
    from sqlalchemy import create_engine
    from sqlalchemy import Table, MetaData, Column
    from sqlalchemy import Integer, String
    engine = create_engine(sqluri)
    metadata = MetaData()
    Table('users_%s' % name,
          metadata,
          Column('id', Integer, primary_key=True),
          Column('username', String(40), unique=True),
          Column('email', String(80)),
          Column('password', String(255)),
          Column('phone', String(40)),
          Column('mobile', String(40)),
          Column('surname', String(40)),
          Column('givenname', String(40)),
          Column('description', String(255)))
    metadata.create_all(engine)


# unfortunately it is not possible in flask_script to add a command with a
# different name and options. So we create an appropriate command class.
class ListResolver(Command):
    """
    Command class to list the available resolvers and the type.
    """
    option_list = (
        Option('-v', '--verbose',
               help="Verbose output - also print the configuration of the resolvers.",
               dest="verbose", action="store_true"),
    )

    def run(self, verbose=False):
        from privacyidea.lib.resolver import get_resolver_list
        resolver_list = get_resolver_list()

        if not verbose:
            for (name, resolver) in resolver_list.items():
                print("{0!s:16} - ({1!s})".format(name, resolver.get("type")))
        else:
            for (name, resolver) in resolver_list.items():
                print("{0!s:16} - ({1!s})".format(name, resolver.get("type")))
                print("."*32)
                data = resolver.get("data", {})
                for (k, v) in data.items():
                    if k.lower() in ["bindpw", "password"]:
                        v = "xxxxx"
                    print("{0!s:>24}: {1!r}".format(k, v))
                print("")


resolver_manager.add_command('list', ListResolver)


def list_realms():
    """
    list the available realms
    """
    from privacyidea.lib.realm import get_realms
    realm_list = get_realms()
    for (name, realm_data) in realm_list.items():
        resolvernames = [x.get("name") for x in realm_data.get("resolver")]
        print("%16s: %s" % (name, resolvernames))


realm_manager.add_command('list', Command(list_realms))


@realm_manager.command
def create(name, resolvers):
    """
    Create a new realm.
    This will create a new realm with the given resolver
    or a comma-separated list of resolvers. An existing realm
    with the same name will be replaced.
    """
    from privacyidea.lib.realm import set_realm
    resolvers = resolvers.split(",")
    set_realm(name, resolvers)


@realm_manager.command
def delete(realm):
    """
    Delete the given realm
    """
    from privacyidea.lib.realm import delete_realm
    delete_realm(realm)


@realm_manager.command
def set_default(realm):
    """
    Set the given realm to default
    """
    from privacyidea.lib.realm import set_default_realm
    set_default_realm(realm)


@realm_manager.command
def clear_default():
    """
    Unset the default realm
    """
    from privacyidea.lib.realm import set_default_realm
    set_default_realm(None)


# Event interface


def list_events():
    """
    List events
    """
    conf = EventConfiguration()
    events = conf.events
    print("{0:7} {4:4} {1:30}\t{2:20}\t{3}".format("Active", "Name", "Module", "Action", "ID"))
    print(90*"=")
    for event in events:
        print("[{0!s:>5}] {4:4} {1:30}\t{2:20}\t{3}".format(event.get("active"),
                                                            event.get("name")[0:30],
                                                            event.get("handlermodule"),
                                                            event.get("action"), event.get("id"),))


event_manager.add_command('list', Command(list_events))


@event_manager.command
def enable(eid):
    """
    enable en event by ID
    """
    r = enable_event(eid)
    print(r)


@event_manager.command
def disable(eid):
    """
    disable an event by ID
    """
    r = enable_event(eid, enable=False)
    print(r)


@event_manager.command
def delete(eid):
    """
    delete an event by ID
    """
    r = delete_event(eid)
    print(r)


@event_manager.command
def e_export(filename=None):
    """
    Export all events to a file
    """
    import pprint
    pp = pprint.PrettyPrinter(indent=4)
    conf = EventConfiguration()
    events = conf.events
    event_str = pp.pformat(events)
    if filename:
        with open(filename, "w") as f:
            f.write(event_str)
    else:
        print(event_str)


@event_manager.command
def e_import(filename, cleanup=False):
    """
    Import the events from a file.
    If 'cleanup' is specified the existing events are deleted before the
    events from the file are imported.
    """
    if cleanup:
        print("Cleanup old events.")
        conf = EventConfiguration()
        events = conf.events
        for event in events:
            name = event.get("name")
            r = delete_event(event.get("id"))
            print("Deleted event '{0!s}' with result {1!s}".format(name, r))

    with open(filename, 'r') as f:
        contents = f.read()

    events = ast.literal_eval(contents)
    for event in events:
        name = event.get("name")
        r = set_event(name, event.get("event"), event.get("handlermodule"),
                      event.get("action"),
                      conditions=event.get("conditions"),
                      ordering=event.get("ordering"),
                      options=event.get("options"),
                      active=event.get("active"),
                      position=event.get("position", "post"))
        print("Added event '{0!s}' with result {1!s}".format(name, r))


# Policy interface

def list_policies():
    """
    list the policies
    """
    pol_cls = PolicyClass()
    policies = pol_cls.list_policies()
    print("Active \t Name \t Scope")
    print(40*"=")
    for policy in policies:
        print("%s \t %s \t %s" % (policy.get("active"), policy.get("name"),
                                  policy.get("scope")))


policy_manager.add_command('list', Command(list_policies))


@policy_manager.command
def enable(name):
    """
    enable a policy by name
    """
    r = enable_policy(name)
    print(r)


@policy_manager.command
def disable(name):
    """
    disable a policy by name
    """
    r = enable_policy(name, False)
    print(r)


@policy_manager.command
def delete(name):
    """
    delete a policy by name
    """
    r = delete_policy(name)
    print(r)


@policy_manager.command
def p_export(filename=None):
    """
    Export all policies to a file.
    If the filename is omitted, the policies are written to stdout.
    """
    import pprint
    pp = pprint.PrettyPrinter(indent=4)
    pol_cls = PolicyClass()
    policies = pol_cls.list_policies()
    pol_str = pp.pformat(policies)
    if filename:
        with open(filename, 'w') as f:
            f.write(pol_str)
    else:
        print(pol_str)


@policy_manager.command
def p_import(filename, cleanup=False):
    """
    Import the policies from a file.
    If 'cleanup' is specified the existing policies are deleted before the
    policies from the file are imported.
    """
    if cleanup:
        print("Cleanup old policies.")
        pol_cls = PolicyClass()
        policies = pol_cls.list_policies()
        for policy in policies:
            name = policy.get("name")
            r = delete_policy(name)
            print("Deleted policy {0!s} with result {1!s}".format(name, r))

    with open(filename, 'r') as f:
        contents = f.read()
    policies = ast.literal_eval(contents)
    for policy in policies:
        name = policy.get("name")
        r = set_policy(name,
                       action=policy.get("action"),
                       active=policy.get("active", True),
                       adminrealm=policy.get("adminrealm"),
                       adminuser=policy.get("adminuser"),
                       check_all_resolvers=policy.get(
                           "check_all_resolvers", False),
                       client=policy.get("client"),
                       # condition=policy.get("condition"),
                       realm=policy.get("realm"),
                       resolver=policy.get("resolver"),
                       scope=policy.get("scope"),
                       time=policy.get("time"),
                       user=policy.get("user"))
        print("Added policy {0!s} with result {1!s}".format(name, r))


@policy_manager.command
def create(name, scope, action, filename=None):
    """
    create a new policy. 'FILENAME' must contain a dictionary and its content
    takes precedence over CLI parameters.
    I.e. if you are specifying a FILENAME,
    the parameters name, scope and action need to be specified, but are ignored.

    Note: This will only create one policy per file.
    """
    if filename:
        try:
            with open(filename, 'r') as f:
                contents = f.read()

            params = ast.literal_eval(contents)

            if params.get("name") and params.get("name") != name:
                print("Found name '{0!s}' in file, will use that instead of "
                      "'{1!s}'.".format(params.get("name"), name))
            else:
                print("name not defined in file, will use the cli value "
                      "{0!s}.".format(name))
                params["name"] = name

            if params.get("scope") and params.get("scope") != scope:
                print("Found scope '{0!s}' in file, will use that instead of "
                      "'{1!s}'.".format(params.get("scope"), scope))
            else:
                print("scope not defined in file, will use the cli value "
                      "{0!s}.".format(scope))
                params["scope"] = scope

            if params.get("action") and params.get("action") != action:
                print("Found action in file: '{0!s}', will use that instead "
                      "of: '{1!s}'.".format(params.get("action"), action))
            else:
                print("action not defined in file, will use the cli value "
                      "{0!s}.".format(action))
                params["action"] = action

            r = set_policy(params.get("name"),
                           scope=params.get("scope"),
                           action=params.get("action"),
                           realm=params.get("realm"),
                           resolver=params.get("resolver"),
                           user=params.get("user"),
                           time=params.get("time"),
                           client=params.get("client"),
                           active=params.get("active", True),
                           adminrealm=params.get("adminrealm"),
                           adminuser=params.get("adminuser"),
                           check_all_resolvers=params.get(
                               "check_all_resolvers", False))
            return r

        except Exception as _e:
            print("Unexpected error: {0!s}".format(sys.exc_info()[1]))

    else:
        r = set_policy(name, scope, action)
        return r


@api_manager.option('-r', '--role',
                    help="The role of the API key can either be "
                         "'admin' or 'validate' to access the admin "
                         "API or the validate API.",
                    default=ROLE.ADMIN)
@api_manager.option('-d', '--days',
                    help='The number of days the access token should be valid.'
                         ' Defaults to 365.',
                    default=365)
@api_manager.option('-R', '--realm',
                    help='The realm of the admin. Defaults to "API"',
                    default="API")
@api_manager.option('-u', '--username',
                    help='The username of the admin.')
def createtoken(role, days, realm, username):
    """
    Create an API authentication token
    for administrative or validate use.
    Possible roles are "admin" or "validate".
    """
    if role not in ["admin", "validate"]:
        print("ERROR: The role must be 'admin' or 'validate'!")
        sys.exit(1)
    username = username or geturandom(hex=True)
    secret = app.config.get("SECRET_KEY")
    authtype = "API"
    validity = timedelta(days=int(days))
    token = jwt.encode({"username": username,
                        "realm": realm,
                        "nonce": geturandom(hex=True),
                        "role": role,
                        "authtype": authtype,
                        "exp": datetime.datetime.utcnow() + validity,
                        "rights": "TODO"},
                       secret)
    print("Username:   {0!s}".format(username))
    print("Realm:      {0!s}".format(realm))
    print("Role:       {0!s}".format(role))
    print("Validity:   {0!s} days".format(days))
    print("Auth-Token: {0!s}".format(token))


def import_tokens(file, tokenrealm=None):
    """
    Import Tokens from a CSV file
    """
    contents = ""
    with open(file, "r") as f:
        contents = f.read()
    tokens = parseOATHcsv(contents)
    tokenrealms = [tokenrealm] if tokenrealm else []
    i = 0
    for serial in tokens:
        i += 1
        print(u"{0!s}/{1!s} Importing token {2!s}".format(i, len(tokens), serial))

        import_token(serial, tokens[serial], tokenrealms=tokenrealm)


token_manager.add_command('import', Command(import_tokens))


if __name__ == '__main__':
    # We add one blank line, to separate the messages from the initialization
    print("""
             _                    _______  _______
   ___  ____(_)  _____ _______ __/  _/ _ \/ __/ _ |
  / _ \/ __/ / |/ / _ `/ __/ // // // // / _// __ |
 / .__/_/ /_/|___/\_,_/\__/\_, /___/____/___/_/ |_|
/_/                       /___/
   """)
    manager.run()