From 409d364383a1777ce0c5ef85fc19b432bce6c6d1 Mon Sep 17 00:00:00 2001 From: Mustafa Baser Date: Fri, 4 Mar 2022 15:47:10 +0300 Subject: [PATCH] feat(jans-linux-setup): refactored argsp (#969) * refactor: jans-linux-setup reusable argument parser * fix: jans-linux-setup installer -u arg * fix: jans-linux-setup fix checking downoaded files * feat: jans-linux-setup download verbosity * feat: jans-linux-setup argparse pkg issue * feat: jans-linux-setup no-prompt * fix: jans-linux-setup update package list * fix: jans-linux-setup update package list * fix: jans-linux-setup logging argparser actions * fix: jans-linux-setup add protobuf to dependency --- jans-linux-setup/jans_setup/install.py | 214 ++++++------ jans-linux-setup/jans_setup/jans_setup.py | 312 ++++++++++-------- .../jans_setup/setup_app/utils/arg_parser.py | 193 ++++++----- .../jans_setup/setup_app/utils/base.py | 14 +- jans-linux-setup/setup.py | 1 + 5 files changed, 396 insertions(+), 338 deletions(-) diff --git a/jans-linux-setup/jans_setup/install.py b/jans-linux-setup/jans_setup/install.py index 90a502134ba..06b7e205ba5 100755 --- a/jans-linux-setup/jans_setup/install.py +++ b/jans-linux-setup/jans_setup/install.py @@ -10,11 +10,12 @@ import json import re +from types import SimpleNamespace from urllib import request from urllib.parse import urljoin from pathlib import Path -maven_base_url = 'https://maven.jans.io/maven/io/jans/' +ssl._create_default_https_context = ssl._create_unverified_context app_versions = { "JANS_APP_VERSION": "1.0.0", @@ -26,10 +27,21 @@ "SETUP_BRANCH": "main", } +app_globals = SimpleNamespace() +app_globals.jetty_dist_string = 'jetty-distribution' +app_globals.jetty_services = ['jans-auth', 'jans-config-api'] +app_globals.package_dependencies = [] + jans_dir = '/opt/jans' app_dir = '/opt/dist/app' jans_app_dir = '/opt/dist/jans' scripts_dir = '/opt/dist/scripts' +jetty_home = '/opt/jans/jetty' +maven_base_url = 'https://maven.jans.io/maven/io/jans/' +jans_zip_file = os.path.join(jans_app_dir, 'jans.zip') +sqlalchemy_zip_file = os.path.join(jans_app_dir, 'sqlalchemy.zip') + +package_installer = shutil.which('apt') or shutil.which('dnf') or shutil.which('yum') or shutil.which('zypper') for d in (jans_dir, app_dir, jans_app_dir, scripts_dir): @@ -38,11 +50,11 @@ parser = argparse.ArgumentParser(description="This script downloads Janssen Server components and fires setup") parser.add_argument('-a', help=argparse.SUPPRESS, action='store_true') -parser.add_argument('-u', help="Use already downloaded components", action='store_true') +parser.add_argument('-use-downloaded', help="Use already downloaded components", action='store_true') parser.add_argument('-upgrade', help="Upgrade Janssen war and jar files", action='store_true') parser.add_argument('-uninstall', help="Uninstall Jans server and removes all files", action='store_true') parser.add_argument('--args', help="Arguments to be passed to setup.py") -parser.add_argument('-n', help="No prompt", action='store_true') +parser.add_argument('-yes', help="No prompt", action='store_true') parser.add_argument('--keep-downloads', help="Keep downloaded files (applicable for uninstallation only)", action='store_true') parser.add_argument('--profile', help="Setup profile", choices=['jans', 'openbanking'], default='jans') parser.add_argument('--jans-app-version', help="Version for Jannses applications") @@ -52,59 +64,54 @@ parser.add_argument('--no-jans-setup', help="Do not extract jans-setup", action='store_true') parser.add_argument('--no-gcs', help="Do not download gcs", action='store_true') parser.add_argument('-setup-dir', help="Setup directory", default=os.path.join(jans_dir, 'jans-setup')) - +parser.add_argument('--force-download', help="Force downloading files", action='store_true') if '-a' in sys.argv: parser.add_argument('--jetty-version', help="Jetty verison. For example 11.0.6") -argsp = parser.parse_args() -setup_dir = argsp.setup_dir -ssl._create_default_https_context = ssl._create_unverified_context - -if argsp.jans_app_version: - app_versions['JANS_APP_VERSION'] = argsp.jans_app_version +def init_installer(): -if argsp.jans_build: - app_versions['JANS_BUILD'] = argsp.jans_build + if app_globals.argsp.jans_app_version: + app_versions['JANS_APP_VERSION'] = app_globals.argsp.jans_app_version -if argsp.setup_branch: - app_versions['SETUP_BRANCH'] = argsp.setup_branch + if app_globals.argsp.jans_build: + app_versions['JANS_BUILD'] = app_globals.argsp.jans_build -jetty_dist_string = 'jetty-distribution' -if getattr(argsp, 'jetty_version', None): - result = re.findall('(\d*).', argsp.jetty_version) - if result and result[0] and result[0].isdigit(): - if int(result[0]) > 9: - jetty_dist_string = 'jetty-home' - app_versions['JETTY_VERSION'] = argsp.jetty_version - else: - print("Can't determine Jetty Version. Continuing with version {}".format(app_versions['JETTY_VERSION'])) + if app_globals.argsp.setup_branch: + app_versions['SETUP_BRANCH'] = app_globals.argsp.setup_branch + if getattr(app_globals.argsp, 'jetty_version', None): + result = re.findall('(\d*).', app_globals.argsp.jetty_version) + if result and result[0] and result[0].isdigit(): + if int(result[0]) > 9: + app_globals.jetty_dist_string = 'jetty-home' + app_versions['JETTY_VERSION'] = app_globals.argsp.jetty_version + else: + print("Can't determine Jetty Version. Continuing with version {}".format(app_versions['JETTY_VERSION'])) -package_installer = shutil.which('apt') or shutil.which('dnf') or shutil.which('yum') or shutil.which('zypper') - -package_dependencies = [] + if app_globals.argsp.profile == 'jans': + app_globals.jetty_services += ['jans-fido2', 'jans-scim', 'jans-eleven'] def check_install_dependencies(): try: from distutils import dist except: - package_dependencies.append('python3-distutils') + app_globals.package_dependencies.append('python3-distutils') try: import ldap3 except: - package_dependencies.append('python3-ldap3') + app_globals.package_dependencies.append('python3-ldap3') - if package_dependencies and not argsp.n: - install_dist = input('Required package(s): {}. Install now? [Y/n] '.format(', '.join(package_dependencies))) + if app_globals.package_dependencies and not app_globals.argsp.yes: + install_dist = input('Required package(s): {}. Install now? [Y/n] '.format(', '.join(app_globals.package_dependencies))) if install_dist.lower().startswith('n'): print("Can't continue...") sys.exit() - os.system('{} install -y {}'.format(package_installer, ' '.join(package_dependencies))) + os.system('{} install -y {}'.format(package_installer, ' '.join(app_globals.package_dependencies))) def extract_subdir(zip_fn, sub_dir, target_dir, zipf=None): @@ -215,17 +222,13 @@ def download_gcs(): shutil.rmtree(tmp_dir) -jans_zip_file = os.path.join(jans_app_dir, 'jans.zip') -sqlalchemy_zip_file = os.path.join(jans_app_dir, 'sqlalchemy.zip') - - def download_files(): setup_url = 'https://github.com/JanssenProject/jans/archive/refs/heads/{}.zip'.format(app_versions['SETUP_BRANCH']) download(setup_url, jans_zip_file) download('https://corretto.aws/downloads/resources/{0}/amazon-corretto-{0}-linux-x64.tar.gz'.format(app_versions['AMAZON_CORRETTO_VERSION']), os.path.join(app_dir, 'amazon-corretto-{0}-linux-x64.tar.gz'.format(app_versions['AMAZON_CORRETTO_VERSION']))) - download('https://repo1.maven.org/maven2/org/eclipse/jetty/{1}/{0}/{1}-{0}.tar.gz'.format(app_versions['JETTY_VERSION'], jetty_dist_string), os.path.join(app_dir,'{1}-{0}.tar.gz'.format(app_versions['JETTY_VERSION'], jetty_dist_string))) + download('https://repo1.maven.org/maven2/org/eclipse/jetty/{1}/{0}/{1}-{0}.tar.gz'.format(app_versions['JETTY_VERSION'], app_globals.jetty_dist_string), os.path.join(app_dir,'{1}-{0}.tar.gz'.format(app_versions['JETTY_VERSION'], app_globals.jetty_dist_string))) download('https://maven.gluu.org/maven/org/gluufederation/jython-installer/{0}/jython-installer-{0}.jar'.format(app_versions['JYTHON_VERSION']), os.path.join(app_dir, 'jython-installer-{0}.jar'.format(app_versions['JYTHON_VERSION']))) download(urljoin(maven_base_url, 'jans-auth-server/{0}{1}/jans-auth-server-{0}{1}.war'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD'])), os.path.join(jans_app_dir, 'jans-auth.war')) download(urljoin(maven_base_url, 'jans-auth-client/{0}{1}/jans-auth-client-{0}{1}-jar-with-dependencies.jar'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD'])), os.path.join(jans_app_dir, 'jans-auth-client-jar-with-dependencies.jar')) @@ -237,27 +240,23 @@ def download_files(): download('https://raw.githubusercontent.com/GluuFederation/gluu-snap/master/facter/facter', os.path.join(jans_app_dir, 'facter')) download('https://github.com/jpadilla/pyjwt/archive/refs/tags/2.3.0.zip', os.path.join(app_dir, 'pyjwt.zip')) - if argsp.profile == 'jans': + if app_globals.argsp.profile == 'jans': download('https://maven.gluu.org/maven/org/gluufederation/opendj/opendj-server-legacy/{0}/opendj-server-legacy-{0}.zip'.format(app_versions['OPENDJ_VERSION']), os.path.join(app_dir, 'opendj-server-legacy-{0}.zip'.format(app_versions['OPENDJ_VERSION']))) download(urljoin(maven_base_url, 'jans-fido2-server/{0}{1}/jans-fido2-server-{0}{1}.war'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD'])), os.path.join(jans_app_dir, 'jans-fido2.war')) download(urljoin(maven_base_url, 'jans-scim-server/{0}{1}/jans-scim-server-{0}{1}.war'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD'])), os.path.join(jans_app_dir, 'jans-scim.war')) download('https://jenkins.jans.io/maven/io/jans/jans-eleven-server/{0}{1}/jans-eleven-server-{0}{1}.war'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD']), os.path.join(jans_app_dir, 'jans-eleven.war')) download('https://www.apple.com/certificateauthority/Apple_WebAuthn_Root_CA.pem', os.path.join(app_dir, 'Apple_WebAuthn_Root_CA.pem')) -jetty_home = '/opt/jans/jetty' -jetty_services = ['jans-auth', 'jans-config-api'] -if argsp.profile == 'jans': - jetty_services += ['jans-fido2', 'jans-scim', 'jans-eleven'] def check_installation(): - if not (os.path.exists(jetty_home) and os.path.join('/opt/jans/jans-setup/setup_app') and ('/etc/jans/conf/jans.properties')): + if not (os.path.exists(jetty_home) and os.path.exists('/etc/jans/conf/jans.properties')): print("Jans server seems not installed") sys.exit() def profile_setup(): - print("Preparing Setup for profile {}".format(argsp.profile)) - profile_dir = os.path.join(setup_dir, argsp.profile) + print("Preparing Setup for profile {}".format(app_globals.argsp.profile)) + profile_dir = os.path.join(app_globals.argsp.setup_dir, app_globals.argsp.profile) replace_dirs = [] if not os.path.exists(profile_dir): print("Profile directory {} does not exist. Exiting ...".format(profile_dir)) @@ -274,11 +273,11 @@ def profile_setup(): replaced_dirs = [] for pdir in replace_dirs: source_dir = os.path.join(profile_dir, pdir) - target_dir = os.path.join(setup_dir, pdir) + target_dir = os.path.join(app_globals.argsp.setup_dir, pdir) replaced_dirs.append(source_dir) if os.path.exists(source_dir) and os.path.exists(target_dir): shutil.rmtree(target_dir) - copy_target = os.path.join(setup_dir, os.path.sep.join(os.path.split(pdir)[:-1])) + copy_target = os.path.join(app_globals.argsp.setup_dir, os.path.sep.join(os.path.split(pdir)[:-1])) print(target_dir) shutil.copytree(source_dir, target_dir) @@ -286,7 +285,7 @@ def profile_setup(): if root.startswith(tuple(replaced_dirs)): continue if files: - target_dir = Path(setup_dir).joinpath(Path(root).relative_to(Path(profile_dir))) + target_dir = Path(app_globals.argsp.setup_dir).joinpath(Path(root).relative_to(Path(profile_dir))) for f in files: if f in ['.profiledirs']: continue @@ -295,19 +294,19 @@ def profile_setup(): shutil.copy(source_file, target_dir) def extract_setup(): - if os.path.exists(setup_dir): - shutil.move(setup_dir, setup_dir + '-back.' + time.ctime()) + if os.path.exists(app_globals.argsp.setup_dir): + shutil.move(app_globals.argsp.setup_dir, app_globals.argsp.setup_dir + '-back.' + time.ctime()) print("Extracting jans-setup package") - extract_subdir(jans_zip_file, 'jans-linux-setup/jans_setup', setup_dir) - extract_subdir(sqlalchemy_zip_file, 'lib/sqlalchemy', os.path.join(setup_dir, 'setup_app/pylib/sqlalchemy')) + extract_subdir(jans_zip_file, 'jans-linux-setup/jans_setup', app_globals.argsp.setup_dir) + extract_subdir(sqlalchemy_zip_file, 'lib/sqlalchemy', os.path.join(app_globals.argsp.setup_dir, 'setup_app/pylib/sqlalchemy')) - target_setup = os.path.join(setup_dir, 'setup.py') + target_setup = os.path.join(app_globals.argsp.setup_dir, 'setup.py') if not os.path.exists(target_setup): - os.symlink(os.path.join(setup_dir, 'jans_setup.py'), target_setup) + os.symlink(os.path.join(app_globals.argsp.setup_dir, 'jans_setup.py'), target_setup) def extract_yaml_files(): - extract_file(jans_zip_file, 'jans-config-api/docs/jans-config-api-swagger.yaml', os.path.join(setup_dir, 'setup_app/data')) - extract_file(jans_zip_file, 'jans-scim/server/src/main/resources/jans-scim-openapi.yaml', os.path.join(setup_dir, 'setup_app/data')) + extract_file(jans_zip_file, 'jans-config-api/docs/jans-config-api-swagger.yaml', os.path.join(app_globals.argsp.setup_dir, 'setup_app/data')) + extract_file(jans_zip_file, 'jans-scim/server/src/main/resources/jans-scim-openapi.yaml', os.path.join(app_globals.argsp.setup_dir, 'setup_app/data')) extract_file(jans_zip_file, 'jans-config-api/server/src/main/resources/log4j2.xml', jans_app_dir) def prepare_jans_cli_package(): @@ -316,7 +315,7 @@ def prepare_jans_cli_package(): def uninstall_jans(): check_installation() - if not argsp.n: + if not app_globals.argsp.yes: print('\033[31m') print("This process is irreversible.") print("You will lose all data related to Janssen Server.") @@ -335,7 +334,7 @@ def uninstall_jans(): print("Please type \033[1m yes \033[0m to uninstall") print("Uninstalling Jannsen Server...") - for service in jetty_services: + for service in app_globals.jetty_services: if os.path.exists(os.path.join(jetty_home, service)): default_fn = os.path.join('/etc/default/', service) if os.path.exists(default_fn): @@ -344,14 +343,14 @@ def uninstall_jans(): print("Stopping", service) os.system('systemctl stop ' + service) - if argsp.profile == 'jans': + if app_globals.argsp.profile == 'jans': print("Stopping OpenDj Server") os.system('/opt/opendj/bin/stop-ds') remove_list = ['/etc/certs', '/etc/jans', '/opt/jans', '/opt/amazon-corretto*', '/opt/jre', '/opt/node*', '/opt/jetty*', '/opt/jython*'] - if argsp.profile == 'jans': + if app_globals.argsp.profile == 'jans': remove_list.append('/opt/opendj') - if not argsp.keep_downloads: + if not app_globals.argsp.keep_downloads: remove_list.append('/opt/dist') for p in remove_list: @@ -360,63 +359,64 @@ def uninstall_jans(): os.system('rm -r -f ' + p) -def main(): +def upgrade(): + check_installation() - check_install_dependencies() + for service in app_globals.jetty_services: + source_fn = os.path.join('/opt/dist/jans', service +'.war') + target_fn = os.path.join(jetty_home, service, 'webapps', service +'.war' ) + if os.path.exists(target_fn): + print("Copying", source_fn, "as", target_fn) + shutil.move(target_fn, target_fn+'-back.' + time.ctime()) + shutil.copy(source_fn, target_fn) + print("Restarting", service) + os.system('systemctl restart ' + service) + + jans_config_api_fn = '/opt/jans/config-api/jans-config-api-runner.jar' + if os.path.exists(jans_config_api_fn): + shutil.move(jans_config_api_fn, jans_config_api_fn + '-back.' + time.ctime()) + source_fn = '/opt/dist/jans/jans-config-api-runner.jar' + print("Copying", source_fn, "as", jans_config_api_fn) + shutil.copy(source_fn, jans_config_api_fn) + print("Restarting jans-config-api") + os.system('systemctl restart jans-config-api') + +def do_install(): + if not app_globals.argsp.no_jans_setup: + extract_setup() + + extract_yaml_files() + + if app_globals.argsp.profile == 'jans': + if not app_globals.argsp.no_gcs: + download_gcs() + else: + profile_setup() - if not (argsp.u or argsp.uninstall): - download_files() + if not app_globals.argsp.no_setup: + print("Launching Janssen Setup") - if argsp.upgrade: + setup_cmd = 'python3 {}/setup.py'.format(app_globals.argsp.setup_dir) - check_installation() + if app_globals.argsp.args: + setup_cmd += ' ' + app_globals.argsp.args - for service in jetty_services: - source_fn = os.path.join('/opt/dist/jans', service +'.war') - target_fn = os.path.join(jetty_home, service, 'webapps', service +'.war' ) - if os.path.exists(target_fn): - print("Copying", source_fn, "as", target_fn) - shutil.move(target_fn, target_fn+'-back.' + time.ctime()) - shutil.copy(source_fn, target_fn) - print("Restarting", service) - os.system('systemctl restart ' + service) + os.system(setup_cmd) - jans_config_api_fn = '/opt/jans/config-api/jans-config-api-runner.jar' - if os.path.exists(jans_config_api_fn): - shutil.move(jans_config_api_fn, jans_config_api_fn + '-back.' + time.ctime()) - source_fn = '/opt/dist/jans/jans-config-api-runner.jar' - print("Copying", source_fn, "as", jans_config_api_fn) - shutil.copy(source_fn, jans_config_api_fn) - print("Restarting jans-config-api") - os.system('systemctl restart jans-config-api') +def main(): + app_globals.argsp = parser.parse_known_args()[0] + init_installer() + check_install_dependencies() - elif argsp.uninstall: + if not (app_globals.argsp.use_downloaded or app_globals.argsp.uninstall): + download_files() + if app_globals.argsp.upgrade: + upgrade() + elif app_globals.argsp.uninstall: uninstall_jans() - else: - - if not argsp.no_jans_setup: - extract_setup() - - extract_yaml_files() - - if argsp.profile == 'jans': - if not argsp.no_gcs: - download_gcs() - else: - profile_setup() - - - if not argsp.no_setup: - print("Launching Janssen Setup") - - setup_cmd = 'python3 {}/setup.py'.format(setup_dir) - - if argsp.args: - setup_cmd += ' ' + argsp.args - - os.system(setup_cmd) + do_install() if __name__ == "__main__": diff --git a/jans-linux-setup/jans_setup/jans_setup.py b/jans-linux-setup/jans_setup/jans_setup.py index c4192943116..ee428175187 100755 --- a/jans-linux-setup/jans_setup/jans_setup.py +++ b/jans-linux-setup/jans_setup/jans_setup.py @@ -11,33 +11,47 @@ import traceback import code import site +import argparse from pathlib import Path from queue import Queue __STATIC_SETUP_DIR__ = '/opt/jans/jans-setup/' -queue = Queue() +os.environ['LC_ALL'] = 'C' +queue = Queue() dir_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(dir_path) -if not (os.path.exists('/opt/dist/jans/jans.zip') or os.path.exists('jans-auth.war')) or '-uninstall' in sys.argv: - import install - install.setup_dir = dir_path - if '-uninstall' in sys.argv: - install.uninstall_jans() - sys.exit() +from setup_app.utils import arg_parser + +import install + +group = arg_parser.parser.add_argument_group(install.parser.description) + +for action in install.parser._actions: + if isinstance(action, argparse._HelpAction): + continue + if isinstance(action, argparse._StoreTrueAction): + arg = group.add_argument(*action.option_strings, action='store_true') else: - print("Downloading Files") - install.download_files() - install.extract_yaml_files() - install.prepare_jans_cli_package() + arg = group.add_argument(*action.option_strings) -os.environ['LC_ALL'] = 'C' -from setup_app.utils.arg_parser import arg_parser + arg.option_strings = action.option_strings + arg.default = action.default + arg.help = action.help + arg.choices = action.choices + arg.required = False + arg.type = action.type -argsp = arg_parser() +argsp = arg_parser.parser.parse_known_args()[0] + +if not (os.path.exists('/opt/dist/jans/jans.zip') or os.path.exists('/opt/dist/jans/jans-auth.war')) or '-uninstall' in sys.argv or argsp.force_download: + + install.app_globals.argsp = argsp + install.setup_dir = dir_path + install.init_installer() # first import paths and make changes if necassary from setup_app import paths @@ -128,166 +142,180 @@ def ami_packaged(): if not os.path.exists(Config.outputFolder): os.makedirs(Config.outputFolder) -# initialize config object -Config.init(paths.INSTALL_DIR) -Config.determine_version() -base.profile = Config.profile -if Config.profile != 'jans': - argsp.t = False - -# we must initilize SetupUtils after initilizing Config -SetupUtils.init() +def main(): -# get setup options from args -setupOptions = get_setup_options() + if '-uninstall' in sys.argv: + install.uninstall_jans() + sys.exit() + elif not os.path.exists('/opt/dist/jans/jans-auth.war'): + print("Downloading Files") + install.download_files() + install.extract_yaml_files() + install.prepare_jans_cli_package() + if argsp.profile != 'jans': + install.profile_setup() + elif not argsp.no_gcs: + install.download_gcs() -terminal_size = shutil.get_terminal_size() -tty_rows = terminal_size.lines -tty_columns = terminal_size.columns + # initialize config object + Config.init(paths.INSTALL_DIR) + Config.determine_version() -# check if we are running in terminal -try: - os.get_terminal_size() -except: - argsp.no_progress = True + base.profile = Config.profile + if Config.profile != 'jans': + argsp.t = False -if not argsp.n: - base.check_resources() + # we must initilize SetupUtils after initilizing Config + SetupUtils.init() -# pass progress indicator to Config object -Config.pbar = jansProgress + # get setup options from args + setupOptions = get_setup_options() -for key in setupOptions: - setattr(Config, key, setupOptions[key]) + terminal_size = shutil.get_terminal_size() + tty_rows = terminal_size.lines + tty_columns = terminal_size.columns -jansInstaller = JansInstaller() -jansInstaller.initialize() + # check if we are running in terminal + try: + os.get_terminal_size() + except: + argsp.no_progress = True + + if not argsp.n: + base.check_resources() + + # pass progress indicator to Config object + Config.pbar = jansProgress + + for key in setupOptions: + setattr(Config, key, setupOptions[key]) + + jansInstaller = JansInstaller() + jansInstaller.initialize() + + print() + detected_os = '{} {}'.format(base.os_type, base.os_version) + if base.snap: + detected_os = 'snap ' + detected_os + print("Installing Janssen Server...\n\nFor more info see:\n {} \n {}\n".format(paths.LOG_FILE, paths.LOG_ERROR_FILE)) + print("Profile : {}".format(Config.profile)) + print("Detected OS : {}".format(detected_os)) + print("Janssen Version : {}".format(Config.oxVersion)) + print("Detected init : {}".format(base.os_initdaemon)) + print("Detected Apache : {}".format(base.determineApacheVersion())) + print() + + setup_loaded = {} + if setupOptions['setup_properties']: + base.logIt('%s Properties found!\n' % setupOptions['setup_properties']) + setup_loaded = propertiesUtils.load_properties(setupOptions['setup_properties']) + elif os.path.isfile(Config.setup_properties_fn): + base.logIt('%s Properties found!\n' % Config.setup_properties_fn) + setup_loaded = propertiesUtils.load_properties(Config.setup_properties_fn) + elif os.path.isfile(Config.setup_properties_fn + '.enc'): + base.logIt('%s Properties found!\n' % Config.setup_properties_fn + '.enc') + setup_loaded = propertiesUtils.load_properties(Config.setup_properties_fn + '.enc') + + if argsp.import_ldif: + if os.path.isdir(argsp.import_ldif): + base.logIt("Found setup LDIF import directory {}".format(argsp.import_ldif)) + else: + base.logIt("The custom LDIF import directory {} does not exist. Exiting...".format(argsp.import_ldif, True, True)) + + collectProperties = CollectProperties() + if os.path.exists(Config.jans_properties_fn): + collectProperties.collect() + collectProperties.save() + Config.installed_instance = True -print() -detected_os = '{} {}'.format(base.os_type, base.os_version) -if base.snap: - detected_os = 'snap ' + detected_os -print("Installing Janssen Server...\n\nFor more info see:\n {} \n {}\n".format(paths.LOG_FILE, paths.LOG_ERROR_FILE)) -print("Profile : {}".format(Config.profile)) -print("Detected OS : {}".format(detected_os)) -print("Janssen Version : {}".format(Config.oxVersion)) -print("Detected init : {}".format(base.os_initdaemon)) -print("Detected Apache : {}".format(base.determineApacheVersion())) -print() - -setup_loaded = {} -if setupOptions['setup_properties']: - base.logIt('%s Properties found!\n' % setupOptions['setup_properties']) - setup_loaded = propertiesUtils.load_properties(setupOptions['setup_properties']) -elif os.path.isfile(Config.setup_properties_fn): - base.logIt('%s Properties found!\n' % Config.setup_properties_fn) - setup_loaded = propertiesUtils.load_properties(Config.setup_properties_fn) -elif os.path.isfile(Config.setup_properties_fn + '.enc'): - base.logIt('%s Properties found!\n' % Config.setup_properties_fn + '.enc') - setup_loaded = propertiesUtils.load_properties(Config.setup_properties_fn + '.enc') - -if argsp.import_ldif: - if os.path.isdir(argsp.import_ldif): - base.logIt("Found setup LDIF import directory {}".format(argsp.import_ldif)) - else: - base.logIt("The custom LDIF import directory {} does not exist. Exiting...".format(argsp.import_ldif, True, True)) + if argsp.csx: + print("Saving collected properties") + collectProperties.save() + sys.exit() -collectProperties = CollectProperties() -if os.path.exists(Config.jans_properties_fn): - collectProperties.collect() - collectProperties.save() - Config.installed_instance = True + if not Config.noPrompt and not Config.installed_instance and not setup_loaded: + propertiesUtils.promptForProperties() - if argsp.csx: - print("Saving collected properties") - collectProperties.save() - sys.exit() + propertiesUtils.check_properties() + # initialize installers, order is important! + jreInstaller = JreInstaller() + jettyInstaller = JettyInstaller() + jythonInstaller = JythonInstaller() -if not Config.noPrompt and not Config.installed_instance and not setup_loaded: - propertiesUtils.promptForProperties() + if Config.profile == 'jans': + openDjInstaller = OpenDjInstaller() + couchbaseInstaller = CouchbaseInstaller() -propertiesUtils.check_properties() -# initialize installers, order is important! -jreInstaller = JreInstaller() -jettyInstaller = JettyInstaller() -jythonInstaller = JythonInstaller() + rdbmInstaller = RDBMInstaller() + httpdinstaller = HttpdInstaller() + jansAuthInstaller = JansAuthInstaller() + configApiInstaller = ConfigApiInstaller() -if Config.profile == 'jans': - openDjInstaller = OpenDjInstaller() - couchbaseInstaller = CouchbaseInstaller() + if Config.profile == 'jans': + fidoInstaller = FidoInstaller() + scimInstaller = ScimInstaller() + elevenInstaller = ElevenInstaller() -rdbmInstaller = RDBMInstaller() -httpdinstaller = HttpdInstaller() -jansAuthInstaller = JansAuthInstaller() -configApiInstaller = ConfigApiInstaller() + jansCliInstaller = JansCliInstaller() -if Config.profile == 'jans': - fidoInstaller = FidoInstaller() - scimInstaller = ScimInstaller() - elevenInstaller = ElevenInstaller() + # oxdInstaller = OxdInstaller() -jansCliInstaller = JansCliInstaller() + rdbmInstaller.packageUtils = packageUtils -# oxdInstaller = OxdInstaller() + if Config.installed_instance: + for service in jansProgress.services: + setattr(Config, service['object'].install_var, service['object'].installed()) -rdbmInstaller.packageUtils = packageUtils + if not argsp.shell: + propertiesUtils.promptForProperties() -if Config.installed_instance: - for service in jansProgress.services: - setattr(Config, service['object'].install_var, service['object'].installed()) + if not (argsp.t or argsp.x) and not Config.addPostSetupService: + print("No service was selected to install. Exiting ...") + sys.exit() - if not argsp.shell: - propertiesUtils.promptForProperties() + def print_or_log(msg): + print(msg) if argsp.x else base.logIt(msg) - if not (argsp.t or argsp.x) and not Config.addPostSetupService: - print("No service was selected to install. Exiting ...") - sys.exit() -def print_or_log(msg): - print(msg) if argsp.x else base.logIt(msg) + if Config.profile == 'jans': + if argsp.t: + testDataLoader = TestDataLoader() + if argsp.t and argsp.x: + print_or_log("Loading test data") + testDataLoader.dbUtils.bind() + testDataLoader.createLdapPw() + configApiInstaller.load_test_data() + testDataLoader.load_test_data() + testDataLoader.deleteLdapPw() + print_or_log("Test data loaded.") -if Config.profile == 'jans': - if argsp.t: - testDataLoader = TestDataLoader() - - if argsp.t and argsp.x: - print_or_log("Loading test data") - testDataLoader.dbUtils.bind() - testDataLoader.createLdapPw() - configApiInstaller.load_test_data() - testDataLoader.load_test_data() - testDataLoader.deleteLdapPw() - print_or_log("Test data loaded.") - - if not argsp.t and argsp.x and argsp.load_config_api_test: - print_or_log("Loading Config Api Test data") - configApiInstaller.load_test_data() - print_or_log("Test data loaded. Exiting ...") - - if argsp.x: - print("Exiting ...") - sys.exit() + if not argsp.t and argsp.x and argsp.load_config_api_test: + print_or_log("Loading Config Api Test data") + configApiInstaller.load_test_data() + print_or_log("Test data loaded. Exiting ...") -Config.installJansCli = Config.installConfigApi or Config.installScimServer + if argsp.x: + print("Exiting ...") + sys.exit() -app_vars = locals().copy() + Config.installJansCli = Config.installConfigApi or Config.installScimServer -if argsp.shell: - code.interact(local=locals()) - sys.exit() + app_vars = locals().copy() -if Config.profile == 'jans': - # re-calculate memory usage - Config.calculate_mem() + if argsp.shell: + code.interact(local=locals()) + sys.exit() -print() -print(jansInstaller) + if Config.profile == 'jans': + # re-calculate memory usage + Config.calculate_mem() -base.current_app.proceed_installation = True + print() + print(jansInstaller) -def main(): + base.current_app.proceed_installation = True if not Config.noPrompt: proceed_prompt = input('Proceed with these values [Y|n] ').lower().strip() diff --git a/jans-linux-setup/jans_setup/setup_app/utils/arg_parser.py b/jans-linux-setup/jans_setup/setup_app/utils/arg_parser.py index 8e0e0f8b468..4feae270ab9 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/arg_parser.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/arg_parser.py @@ -1,87 +1,110 @@ import argparse -def arg_parser(): - parser_description='''Use setup.py to configure your Jans Server and to add initial data required for - oxAuth and oxTrust to start. If setup.properties is found in this folder, these - properties will automatically be used instead of the interactive setup. - ''' - - parser = argparse.ArgumentParser(description=parser_description) - parser.add_argument('-c', help="Use command line instead of tui", action='store_true') - parser.add_argument('-d', help="Installation directory") - parser.add_argument('-f', help="Specify setup.properties file") - parser.add_argument('-n', help="No interactive prompt before install starts. Run with -f", action='store_true') - parser.add_argument('-N', '--no-httpd', help="No apache httpd server", action='store_true') - parser.add_argument('-u', help="Update hosts file with IP address / hostname", action='store_true') - parser.add_argument('-w', help="Get the development head war files", action='store_true') - parser.add_argument('-t', help="Load test data", action='store_true') - parser.add_argument('-x', help="Load test data and exit", action='store_true') - parser.add_argument('-csx', help="Collect setup properties, save and exit", action='store_true') - parser.add_argument('-stm', '--enable-scim-test-mode', help="Enable Scim Test Mode", action='store_true') - parser.add_argument('--allow-pre-released-features', help="Enable options to install experimental features, not yet officially supported", action='store_true') - parser.add_argument('--import-ldif', help="Render ldif templates from directory and import them in LDAP") - parser.add_argument('--listen_all_interfaces', help="Allow the LDAP server to listen on all server interfaces", action='store_true') - - ldap_group = parser.add_mutually_exclusive_group() - ldap_group.add_argument('--remote-ldap', help="Enables using remote LDAP server", action='store_true') - ldap_group.add_argument('--disable-local-ldap', help="Disables installing local LDAP server", action='store_true') - - rdbm_group = parser.add_mutually_exclusive_group() - rdbm_group.add_argument('-remote-rdbm', choices=['mysql', 'pgsql', 'spanner'], help="Enables using remote RDBM server") - rdbm_group.add_argument('-local-rdbm', choices=['mysql', 'pgsql'], help="Enables installing/configuring local RDBM server") - - parser.add_argument('-rdbm-user', help="RDBM username") - parser.add_argument('-rdbm-password', help="RDBM password") - parser.add_argument('-rdbm-port', help="RDBM port") - parser.add_argument('-rdbm-db', help="RDBM database") - parser.add_argument('-rdbm-host', help="RDBM host") - - parser.add_argument('--remote-couchbase', help="Enables using remote couchbase server", action='store_true') - parser.add_argument('--local-couchbase', help="Enables installing couchbase server", action='store_true') - parser.add_argument('-couchbase-admin-user', help="Couchbase admin user") - parser.add_argument('-couchbase-admin-password', help="Couchbase admin user password") - parser.add_argument('-couchbase-bucket-prefix', help="Set prefix for couchbase buckets", default='jans') - parser.add_argument('-couchbase-hostname', help="Remote couchbase server hostname") - - parser.add_argument('--no-data', help="Do not import any data to database backend, used for clustering", action='store_true') - parser.add_argument('--no-jsauth', help="Do not install OAuth2 Authorization Server", action='store_true') - parser.add_argument('-ip-address', help="Used primarily by Apache httpd for the Listen directive") - parser.add_argument('-host-name', help="Internet-facing FQDN that is used to generate certificates and metadata.") - parser.add_argument('-org-name', help="Organization name field used for generating X.509 certificates") - parser.add_argument('-email', help="Email address for support at your organization used for generating X.509 certificates") - parser.add_argument('-city', help="City field used for generating X.509 certificates") - parser.add_argument('-state', help="State field used for generating X.509 certificates") - parser.add_argument('-country', help="Two letters country coude used for generating X.509 certificates") - parser.add_argument('-ldap-admin-password', help="Used as the LDAP directory manager password") - parser.add_argument('-admin-password', help="Used as the Administrator password") - parser.add_argument('-jans-max-mem', help="Total memory (in KB) to be used by Jannses Server") - parser.add_argument('-properties-password', help="Encoded setup.properties file password") - parser.add_argument('--no-config-api', help="Do not install Jans Auth Config Api", action='store_true') - parser.add_argument('--disable-config-api-security', help="Turn off oauth2 security validation for jans-config-api", action='store_true') - - #parser.add_argument('--install-oxd', help="Install Oxd Server", action='store_true') - parser.add_argument('--no-scim', help="Do not install Scim Server", action='store_true') - parser.add_argument('--no-fido2', help="Do not install Fido2 Server", action='store_true') - parser.add_argument('--install-eleven', help="Install Eleven Server", action='store_true') - #parser.add_argument('--oxd-use-jans-storage', help="Use Jans Storage for Oxd Server", action='store_true') - parser.add_argument('--load-config-api-test', help="Load Config Api Test Data", action='store_true') - - #parser.add_argument('--generate-oxd-certificate', help="Generate certificate for oxd based on hostname", action='store_true') - parser.add_argument('--shell', help="Drop into interactive shell before starting installation", action='store_true') - parser.add_argument('-config-patch-creds', help="password:username for downloading auto test ciba password") - parser.add_argument('--dump-config-on-error', help="Dump configuration on error", action='store_true') - parser.add_argument('--no-progress', help="Use simple progress", action='store_true') - - # spanner options - parser.add_argument('-spanner-project', help="Spanner project name") - parser.add_argument('-spanner-instance', help="Spanner instance name") - parser.add_argument('-spanner-database', help="Spanner database name") - spanner_cred_group = parser.add_mutually_exclusive_group() - spanner_cred_group.add_argument('-spanner-emulator-host', help="Use Spanner emulator host") - spanner_cred_group.add_argument('-google-application-credentials', help="Path to Google application credentials json file") - - parser.add_argument('-approved-issuer', help="Api Approved Issuer") - - argsp = parser.parse_args() - - return argsp +from setup_app.utils import base + +parser_description='''Use this script to configure your Jans Server and to add initial data required for +oxAuth and oxTrust to start. If setup.properties is found in this folder, these +properties will automatically be used instead of the interactive setup. +''' + +parser = argparse.ArgumentParser(description=parser_description) +parser.add_argument('-c', help="Use command line instead of tui", action='store_true') +parser.add_argument('-d', help="Installation directory") +parser.add_argument('-f', help="Specify setup.properties file") +parser.add_argument('-n', help="No interactive prompt before install starts. Run with -f", action='store_true') +parser.add_argument('-N', '--no-httpd', help="No apache httpd server", action='store_true') +parser.add_argument('-u', help="Update hosts file with IP address / hostname", action='store_true') +parser.add_argument('-w', help="Get the development head war files", action='store_true') +parser.add_argument('-t', help="Load test data", action='store_true') +parser.add_argument('-x', help="Load test data and exit", action='store_true') +parser.add_argument('-csx', help="Collect setup properties, save and exit", action='store_true') +parser.add_argument('-stm', '--enable-scim-test-mode', help="Enable Scim Test Mode", action='store_true') +parser.add_argument('--allow-pre-released-features', help="Enable options to install experimental features, not yet officially supported", action='store_true') +parser.add_argument('--import-ldif', help="Render ldif templates from directory and import them in LDAP") +parser.add_argument('--listen_all_interfaces', help="Allow the LDAP server to listen on all server interfaces", action='store_true') + +ldap_group = parser.add_mutually_exclusive_group() +ldap_group.add_argument('--remote-ldap', help="Enables using remote LDAP server", action='store_true') +ldap_group.add_argument('--disable-local-ldap', help="Disables installing local LDAP server", action='store_true') + +rdbm_group = parser.add_mutually_exclusive_group() +rdbm_group.add_argument('-remote-rdbm', choices=['mysql', 'pgsql', 'spanner'], help="Enables using remote RDBM server") +rdbm_group.add_argument('-local-rdbm', choices=['mysql', 'pgsql'], help="Enables installing/configuring local RDBM server") + +parser.add_argument('-rdbm-user', help="RDBM username") +parser.add_argument('-rdbm-password', help="RDBM password") +parser.add_argument('-rdbm-port', help="RDBM port") +parser.add_argument('-rdbm-db', help="RDBM database") +parser.add_argument('-rdbm-host', help="RDBM host") + +parser.add_argument('--remote-couchbase', help="Enables using remote couchbase server", action='store_true') +parser.add_argument('--local-couchbase', help="Enables installing couchbase server", action='store_true') +parser.add_argument('-couchbase-admin-user', help="Couchbase admin user") +parser.add_argument('-couchbase-admin-password', help="Couchbase admin user password") +parser.add_argument('-couchbase-bucket-prefix', help="Set prefix for couchbase buckets", default='jans') +parser.add_argument('-couchbase-hostname', help="Remote couchbase server hostname") + +parser.add_argument('--no-data', help="Do not import any data to database backend, used for clustering", action='store_true') +parser.add_argument('--no-jsauth', help="Do not install OAuth2 Authorization Server", action='store_true') +parser.add_argument('-ip-address', help="Used primarily by Apache httpd for the Listen directive") +parser.add_argument('-host-name', help="Internet-facing FQDN that is used to generate certificates and metadata.") +parser.add_argument('-org-name', help="Organization name field used for generating X.509 certificates") +parser.add_argument('-email', help="Email address for support at your organization used for generating X.509 certificates") +parser.add_argument('-city', help="City field used for generating X.509 certificates") +parser.add_argument('-state', help="State field used for generating X.509 certificates") +parser.add_argument('-country', help="Two letters country coude used for generating X.509 certificates") +parser.add_argument('-ldap-admin-password', help="Used as the LDAP directory manager password") +parser.add_argument('-admin-password', help="Used as the Administrator password") +parser.add_argument('-jans-max-mem', help="Total memory (in KB) to be used by Jannses Server") +parser.add_argument('-properties-password', help="Encoded setup.properties file password") +parser.add_argument('--no-config-api', help="Do not install Jans Auth Config Api", action='store_true') +parser.add_argument('--disable-config-api-security', help="Turn off oauth2 security validation for jans-config-api", action='store_true') + +#parser.add_argument('--install-oxd', help="Install Oxd Server", action='store_true') +parser.add_argument('--no-scim', help="Do not install Scim Server", action='store_true') +parser.add_argument('--no-fido2', help="Do not install Fido2 Server", action='store_true') +parser.add_argument('--install-eleven', help="Install Eleven Server", action='store_true') +#parser.add_argument('--oxd-use-jans-storage', help="Use Jans Storage for Oxd Server", action='store_true') +parser.add_argument('--load-config-api-test', help="Load Config Api Test Data", action='store_true') + +#parser.add_argument('--generate-oxd-certificate', help="Generate certificate for oxd based on hostname", action='store_true') +parser.add_argument('--shell', help="Drop into interactive shell before starting installation", action='store_true') +parser.add_argument('-config-patch-creds', help="password:username for downloading auto test ciba password") +parser.add_argument('--dump-config-on-error', help="Dump configuration on error", action='store_true') +parser.add_argument('--no-progress', help="Use simple progress", action='store_true') + +# spanner options +parser.add_argument('-spanner-project', help="Spanner project name") +parser.add_argument('-spanner-instance', help="Spanner instance name") +parser.add_argument('-spanner-database', help="Spanner database name") +spanner_cred_group = parser.add_mutually_exclusive_group() +spanner_cred_group.add_argument('-spanner-emulator-host', help="Use Spanner emulator host") +spanner_cred_group.add_argument('-google-application-credentials', help="Path to Google application credentials json file") + +parser.add_argument('-approved-issuer', help="Api Approved Issuer") + +def add_to_me(you): + + base.logIt("Adding actions from parser: '{}'".format(you.description)) + group = parser.add_argument_group(you.description) + + for action in you._actions: + base.logIt("Adding action '{}'".format(' '.join(action.option_strings))) + if isinstance(action, argparse._HelpAction): + continue + if isinstance(action, argparse._StoreTrueAction): + arg = group.add_argument(*action.option_strings, action='store_true') + else: + arg = group.add_argument(*action.option_strings) + + arg.option_strings = action.option_strings + arg.default = action.default + arg.help = action.help + arg.choices = action.choices + arg.required = False + arg.type = action.type + + +def get_parser(): + argsp = parser.parse_known_args() + return argsp[0] diff --git a/jans-linux-setup/jans_setup/setup_app/utils/base.py b/jans-linux-setup/jans_setup/setup_app/utils/base.py index c53a3085a6c..cb61976c116 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/base.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/base.py @@ -297,21 +297,27 @@ def find_script_names(ldif_file): return name_list -def download(url, dst): +def download(url, dst, verbose=False): pardir, fn = os.path.split(dst) if not os.path.exists(pardir): logIt("Creating driectory", pardir) os.makedirs(pardir) - logIt("Downloading {} to {}".format(url, dst)) + + def mylog(logs): + logIt(logs) + if verbose: + print(logs) + + mylog("Downloading {} to {}".format(url, dst)) download_tries = 1 while download_tries < 4: try: result = urlretrieve(url, dst) f_size = result[1].get('Content-Length','0') - logIt("Download size: {} bytes".format(f_size)) + mylog("Download size: {} bytes".format(f_size)) time.sleep(0.1) except: - logIt("Error downloading {}. Download will be re-tried once more".format(url)) + mylog("Error downloading {}. Download will be re-tried once more".format(url)) download_tries += 1 time.sleep(1) else: diff --git a/jans-linux-setup/setup.py b/jans-linux-setup/setup.py index b5714b806e2..4b94dec6f6d 100644 --- a/jans-linux-setup/setup.py +++ b/jans-linux-setup/setup.py @@ -40,6 +40,7 @@ def find_version(*file_paths): "ruamel.yaml>=0.16.5", "sqlalchemy==1.3.23", "cryptography==36.0.1", + "protobuf", "google-cloud-spanner==3.13.0", "ldap3", "PyMySQL",