From 39ac37861fb377dea7962e77bc5043278b2b3d4c Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 29 Jan 2022 10:44:33 +0100 Subject: [PATCH] Added option to enable reconnect Added option to perform dry run of updater Added possibility to exclude files from updater --- cps/__init__.py | 5 +++++ cps/admin.py | 14 +++++++++++- cps/cli.py | 11 +++++++-- cps/updater.py | 60 ++++++++++++++++++++++++++++++++----------------- cps/web.py | 7 ------ exclude.txt | 0 6 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 exclude.txt diff --git a/cps/__init__.py b/cps/__init__.py index 34ccf4383d..2bfee12e49 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -186,4 +186,9 @@ def get_timezone(): from .updater import Updater updater_thread = Updater() + +# Perform dry run of updater and exit afterwards +if cli.dry_run: + updater_thread.dry_run() + sys.exit(0) updater_thread.start() diff --git a/cps/admin.py b/cps/admin.py index c1931ea47e..814a455d18 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -39,7 +39,7 @@ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text -from . import constants, logger, helper, services +from . import constants, logger, helper, services, cli from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, kobo_sync_status from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ valid_email, check_username @@ -158,6 +158,18 @@ def shutdown(): return json.dumps(showtext), 400 +# method is available without login and not protected by CSRF to make it easy reachable, is per default switched of +# needed for docker applications, as changes on metadata.db from host are not visible to application +@admi.route("/reconnect", methods=['GET']) +def reconnect(): + if cli.args.r: + calibre_db.reconnect_db(config, ub.app_DB_path) + return json.dumps({}) + else: + log.debug("'/reconnect' was accessed but is not enabled") + abort(404) + + @admi.route("/admin/view") @login_required @admin_required diff --git a/cps/cli.py b/cps/cli.py index 31ea84176f..a63d7282ba 100644 --- a/cps/cli.py +++ b/cps/cli.py @@ -40,12 +40,15 @@ def version_info(): help='path and name to SSL certfile, e.g. /opt/test.cert, works only in combination with keyfile') parser.add_argument('-k', metavar='path', help='path and name to SSL keyfile, e.g. /opt/test.key, works only in combination with certfile') -parser.add_argument('-v', '--version', action='version', help='Shows version number and exits Calibre-web', +parser.add_argument('-v', '--version', action='version', help='Shows version number and exits Calibre-Web', version=version_info()) parser.add_argument('-i', metavar='ip-address', help='Server IP-Address to listen') -parser.add_argument('-s', metavar='user:pass', help='Sets specific username to new password') +parser.add_argument('-s', metavar='user:pass', help='Sets specific username to new password and exits Calibre-Web') parser.add_argument('-f', action='store_true', help='Flag is depreciated and will be removed in next version') parser.add_argument('-l', action='store_true', help='Allow loading covers from localhost') +parser.add_argument('-d', action='store_true', help='Dry run of updater to check file permissions in advance ' + 'and exits Calibre-Web') +parser.add_argument('-r', action='store_true', help='Enable public database reconnect route under /reconnect') args = parser.parse_args() settingspath = args.p or os.path.join(_CONFIG_DIR, "app.db") @@ -78,6 +81,9 @@ def version_info(): if args.k == "": keyfilepath = "" + +# dry run updater +dry_run = args.d or None # load covers from localhost allow_localhost = args.l or None # handle and check ip address argument @@ -106,3 +112,4 @@ def version_info(): if args.f: print("Warning: -f flag is depreciated and will be removed in next version") + diff --git a/cps/updater.py b/cps/updater.py index 9090263fbf..1e11ff78b5 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -53,12 +53,10 @@ class Updater(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.paused = False - # self.pause_cond = threading.Condition(threading.Lock()) self.can_run = threading.Event() self.pause() self.status = -1 self.updateIndex = None - # self.run() def get_current_version_info(self): if config.config_updatechannel == constants.UPDATE_STABLE: @@ -85,15 +83,15 @@ def do_work(self): log.debug(u'Extracting zipfile') tmp_dir = gettempdir() z.extractall(tmp_dir) - foldername = os.path.join(tmp_dir, z.namelist()[0])[:-1] - if not os.path.isdir(foldername): + folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1] + if not os.path.isdir(folder_name): self.status = 11 log.info(u'Extracted contents of zipfile not found in temp folder') self.pause() return False self.status = 4 log.debug(u'Replacing files') - if self.update_source(foldername, constants.BASE_DIR): + if self.update_source(folder_name, constants.BASE_DIR): self.status = 6 log.debug(u'Preparing restart of server') time.sleep(2) @@ -184,7 +182,7 @@ def reduce_files(cls, remove_items, exclude_items): return rf @classmethod - def check_permissions(cls, root_src_dir, root_dst_dir): + def check_permissions(cls, root_src_dir, root_dst_dir, logfunction): access = True remove_path = len(root_src_dir) + 1 for src_dir, __, files in os.walk(root_src_dir): @@ -193,7 +191,7 @@ def check_permissions(cls, root_src_dir, root_dst_dir): if not os.path.isdir(root_dir): # root_dir.lstrip(os.sep).startswith('.') or continue if not os.access(root_dir, os.R_OK|os.W_OK): - log.debug("Missing permissions for {}".format(root_dir)) + logfunction("Missing permissions for {}".format(root_dir)) access = False for file_ in files: curr_file = os.path.join(root_dir, file_) @@ -201,7 +199,7 @@ def check_permissions(cls, root_src_dir, root_dst_dir): if not os.path.isfile(curr_file): # or curr_file.startswith('.'): continue if not os.access(curr_file, os.R_OK|os.W_OK): - log.debug("Missing permissions for {}".format(curr_file)) + logfunction("Missing permissions for {}".format(curr_file)) access = False return access @@ -258,18 +256,10 @@ def moveallfiles(cls, root_src_dir, root_dst_dir): def update_source(self, source, destination): # destination files old_list = list() - exclude = ( - os.sep + 'app.db', os.sep + 'calibre-web.log1', os.sep + 'calibre-web.log2', os.sep + 'gdrive.db', - os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json', - os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv', - os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2', - os.sep + '.calibre-web.log.swp', os.sep + '_sqlite3.so', os.sep + 'cps' + os.sep + '.HOMEDIR', - os.sep + 'gmail.json' - ) - additional_path = self.is_venv() + exclude = self._add_excluded_files(log.info) + additional_path =self.is_venv() if additional_path: - exclude = exclude + (additional_path,) - + exclude.append(additional_path) # check if we are in a package, rename cps.py to __init__.py if constants.HOME_CONFIG: shutil.move(os.path.join(source, 'cps.py'), os.path.join(source, '__init__.py')) @@ -293,7 +283,7 @@ def update_source(self, source, destination): remove_items = self.reduce_dirs(rf, new_list) - if self.check_permissions(source, destination): + if self.check_permissions(source, destination, log.debug): self.moveallfiles(source, destination) for item in remove_items: @@ -332,6 +322,12 @@ def _stable_version_info(cls): log.debug("Stable version: {}".format(constants.STABLE_VERSION)) return constants.STABLE_VERSION # Current version + @classmethod + def dry_run(cls): + cls._add_excluded_files(print) + cls.check_permissions(constants.BASE_DIR, constants.BASE_DIR, print) + print("\n*** Finished ***") + @staticmethod def _populate_parent_commits(update_data, status, locale, tz, parents): try: @@ -391,6 +387,30 @@ def _load_nightly_data(repository_url, commit, status): status['message'] = _(u'General error') return status, update_data + @staticmethod + def _add_excluded_files(logfunction): + excluded_files = [ + os.sep + 'app.db', os.sep + 'calibre-web.log1', os.sep + 'calibre-web.log2', os.sep + 'gdrive.db', + os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json', + os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv', + os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2', + os.sep + '.calibre-web.log.swp', os.sep + '_sqlite3.so', os.sep + 'cps' + os.sep + '.HOMEDIR', + os.sep + 'gmail.json', os.sep + 'exclude.txt' + ] + try: + with open(os.path.join(constants.BASE_DIR, "exclude.txt"), "r") as f: + lines = f.readlines() + for line in lines: + proccessed_line = line.strip("\n\r ").strip("\"'").lstrip("\\/ ").\ + replace("\\", os.sep).replace("/", os.sep) + if os.path.exists(os.path.join(constants.BASE_DIR, proccessed_line)): + excluded_files.append(os.sep + proccessed_line) + else: + logfunction("File list for updater: {} not found".format(line)) + except (PermissionError, FileNotFoundError): + logfunction("Excluded file list for updater not found, or not accessible") + return excluded_files + def _nightly_available_updates(self, request_method, locale): tz = datetime.timedelta(seconds=time.timezone if (time.localtime().tm_isdst == 0) else time.altzone) if request_method == "GET": diff --git a/cps/web.py b/cps/web.py index 7ecb7e0e6a..018c000247 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1104,13 +1104,6 @@ def get_tasks_status(): return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") -# method is available without login and not protected by CSRF to make it easy reachable -@app.route("/reconnect", methods=['GET']) -def reconnect(): - calibre_db.reconnect_db(config, ub.app_DB_path) - return json.dumps({}) - - # ################################### Search functions ################################################################ @web.route("/search", methods=["GET"]) diff --git a/exclude.txt b/exclude.txt new file mode 100644 index 0000000000..e69de29bb2