From 933b7d9e1e43d4f5ee76e0eb9bb6bce0b80779d5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 12:40:59 +0200 Subject: [PATCH 1/7] SyncServer - handle 500 error in Gdrive init Introduced recoverable exception ResumableError --- .../modules/sync_server/providers/gdrive.py | 11 +++++-- openpype/modules/sync_server/sync_server.py | 32 +++++++++++-------- openpype/modules/sync_server/utils.py | 5 +++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/openpype/modules/sync_server/providers/gdrive.py b/openpype/modules/sync_server/providers/gdrive.py index f1ea24f6011..b67e5a6cfa1 100644 --- a/openpype/modules/sync_server/providers/gdrive.py +++ b/openpype/modules/sync_server/providers/gdrive.py @@ -7,7 +7,7 @@ from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from openpype.api import Logger from openpype.api import get_system_settings -from ..utils import time_function +from ..utils import time_function, ResumableError import time @@ -63,7 +63,14 @@ def __init__(self, project_name, site_name, tree=None, presets=None): return self.service = self._get_gd_service() - self.root = self._prepare_root_info() + try: + self.root = self._prepare_root_info() + except errors.HttpError: + log.warning("HttpError in sync loop, " + "trying next loop", + exc_info=True) + raise ResumableError + self._tree = tree self.active = True diff --git a/openpype/modules/sync_server/sync_server.py b/openpype/modules/sync_server/sync_server.py index e97c0e8844a..42769ceec29 100644 --- a/openpype/modules/sync_server/sync_server.py +++ b/openpype/modules/sync_server/sync_server.py @@ -8,7 +8,7 @@ from .providers import lib from openpype.lib import PypeLogger -from .utils import SyncStatus +from .utils import SyncStatus, ResumableError log = PypeLogger().get_logger("SyncServer") @@ -266,8 +266,8 @@ async def sync_loop(self): Returns: """ - try: - while self.is_running and not self.module.is_paused(): + while self.is_running and not self.module.is_paused(): + try: import time start_time = None self.module.set_sync_project_settings() # clean cache @@ -385,16 +385,22 @@ async def sync_loop(self): duration = time.time() - start_time log.debug("One loop took {:.2f}s".format(duration)) await asyncio.sleep(self.module.get_loop_delay(collection)) - except ConnectionResetError: - log.warning("ConnectionResetError in sync loop, trying next loop", - exc_info=True) - except CancelledError: - # just stopping server - pass - except Exception: - self.stop() - log.warning("Unhandled exception in sync loop, stopping server", - exc_info=True) + + except ConnectionResetError: + log.warning("ConnectionResetError in sync loop, " + "trying next loop", + exc_info=True) + except CancelledError: + # just stopping server + pass + except ResumableError: + log.warning("ResumableError in sync loop, " + "trying next loop", + exc_info=True) + except Exception: + self.stop() + log.warning("Unhandled exception in sync loop, stopping server", + exc_info=True) def stop(self): """Sets is_running flag to false, 'check_shutdown' shuts server down""" diff --git a/openpype/modules/sync_server/utils.py b/openpype/modules/sync_server/utils.py index 36f3444399f..fa6e63b029c 100644 --- a/openpype/modules/sync_server/utils.py +++ b/openpype/modules/sync_server/utils.py @@ -3,6 +3,11 @@ log = Logger().get_logger("SyncServer") +class ResumableError(Exception): + """Error which could be temporary, skip current loop, try next time""" + pass + + class SyncStatus: DO_NOTHING = 0 DO_UPLOAD = 1 From 9a8b9c1f74085caabcec25778c5fcbb8517fc42f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 12:42:30 +0200 Subject: [PATCH 2/7] SyncServer - bump up default of processed files in one loop for local_drive --- openpype/modules/sync_server/providers/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/providers/lib.py b/openpype/modules/sync_server/providers/lib.py index 58947e115db..01a5d50ba51 100644 --- a/openpype/modules/sync_server/providers/lib.py +++ b/openpype/modules/sync_server/providers/lib.py @@ -92,4 +92,4 @@ class and batch limit. # 7 denotes number of files that could be synced in single loop - learned by # trial and error factory.register_provider('gdrive', GDriveHandler, 7) -factory.register_provider('local_drive', LocalDriveHandler, 10) +factory.register_provider('local_drive', LocalDriveHandler, 50) From 9d88f633321de14afd66269e360c873f29ba2930 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 13:02:30 +0200 Subject: [PATCH 3/7] SyncServer - remember last sort in Sync Queue app --- openpype/modules/sync_server/tray/models.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/tray/models.py b/openpype/modules/sync_server/tray/models.py index ffd81a1588c..bfe4db32f36 100644 --- a/openpype/modules/sync_server/tray/models.py +++ b/openpype/modules/sync_server/tray/models.py @@ -170,6 +170,8 @@ def sort(self, index, order): Sort is happening on a DB side, model is reset, db queried again. + It remembers one last sort, adds it as secondary after new sort. + Args: index (int): column index order (int): 0| @@ -184,7 +186,17 @@ def sort(self, index, order): else: order = -1 - self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1} + backup_sort = dict(self.sort) + + self.sort = {self.SORT_BY_COLUMN[index]: order} # reset + # add last one + for key, val in backup_sort.items(): + if key != '_id': + self.sort[key] = val + break + # add default one + self.sort['_id'] = 1 + self.query = self.get_query() # import json # log.debug(json.dumps(self.query, indent=4).\ From d8d241ac5a5f4623a9c70526bd118518195241e7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 13:03:33 +0200 Subject: [PATCH 4/7] SyncServer - change title of window --- openpype/modules/sync_server/tray/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/tray/app.py b/openpype/modules/sync_server/tray/app.py index d91ba763359..2538675c513 100644 --- a/openpype/modules/sync_server/tray/app.py +++ b/openpype/modules/sync_server/tray/app.py @@ -78,7 +78,7 @@ def __init__(self, sync_server, parent=None): layout.addWidget(footer) self.setLayout(body_layout) - self.setWindowTitle("Sync Server") + self.setWindowTitle("Sync Queue") self.projects.project_changed.connect( lambda: repres.table_view.model().set_project( From 8b570c7c822b97cd569d0fade2a1515160eb7b37 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 13:33:43 +0200 Subject: [PATCH 5/7] SyncServer - change title of window --- openpype/modules/sync_server/sync_server.py | 18 +++++++++++++++++- .../modules/sync_server/sync_server_module.py | 8 ++++++++ openpype/modules/sync_server/tray/widgets.py | 3 +++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/sync_server.py b/openpype/modules/sync_server/sync_server.py index 42769ceec29..9c85dd497a6 100644 --- a/openpype/modules/sync_server/sync_server.py +++ b/openpype/modules/sync_server/sync_server.py @@ -232,6 +232,7 @@ def __init__(self, module): self.loop = None self.is_running = False self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=3) + self.timer = None def run(self): self.is_running = True @@ -384,7 +385,11 @@ async def sync_loop(self): duration = time.time() - start_time log.debug("One loop took {:.2f}s".format(duration)) - await asyncio.sleep(self.module.get_loop_delay(collection)) + + delay = self.module.get_loop_delay(collection) + log.debug("Waiting for {} seconds to new loop".format(delay)) + self.timer = asyncio.create_task(self.run_timer(delay)) + await asyncio.gather(self.timer) except ConnectionResetError: log.warning("ConnectionResetError in sync loop, " @@ -423,6 +428,17 @@ async def check_shutdown(self): await asyncio.sleep(0.07) self.loop.stop() + async def run_timer(self, delay): + """Wait for 'delay' seconds to start next loop""" + await asyncio.sleep(delay) + + def reset_timer(self): + """Called when waiting for next loop should be skipped""" + log.debug("Resetting timer") + if self.timer: + self.timer.cancel() + self.timer = None + def _working_sites(self, collection): if self.module.is_project_paused(collection): log.debug("Both sites same, skipping") diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 59c37877891..ca4033b3857 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -401,6 +401,14 @@ def get_remote_site(self, project_name): return remote_site + def reset_timer(self): + """ + Called when waiting for next loop should be skipped. + + In case of user's involvement (reset site), start that right away. + """ + self.sync_server_thread.reset_timer() + """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): diff --git a/openpype/modules/sync_server/tray/widgets.py b/openpype/modules/sync_server/tray/widgets.py index 21236dc64a5..106fc4b8a87 100644 --- a/openpype/modules/sync_server/tray/widgets.py +++ b/openpype/modules/sync_server/tray/widgets.py @@ -316,6 +316,7 @@ def _add_site(self, selected_ids=None, site_name=None): representation_id)) except ValueError as exp: self.message_generated.emit("Error {}".format(str(exp))) + self.sync_server.reset_timer() def _remove_site(self, selected_ids=None, site_name=None): """ @@ -343,6 +344,7 @@ def _remove_site(self, selected_ids=None, site_name=None): self.model.refresh( load_records=self.model._rec_loaded) + self.sync_server.reset_timer() def _reset_site(self, selected_ids=None, site_name=None): """ @@ -368,6 +370,7 @@ def _reset_site(self, selected_ids=None, site_name=None): self.model.refresh( load_records=self.model._rec_loaded) + self.sync_server.reset_timer() def _open_in_explorer(self, selected_ids=None, site_name=None): log.debug("Open in Explorer {}:{}".format(selected_ids, site_name)) From 1cb47023a103b5e09a19aac1f7eb36a53c8ae7f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 14:02:29 +0200 Subject: [PATCH 6/7] Hound --- openpype/modules/sync_server/sync_server.py | 2 +- openpype/modules/sync_server/tray/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/sync_server/sync_server.py b/openpype/modules/sync_server/sync_server.py index 9c85dd497a6..9b305a1b2e7 100644 --- a/openpype/modules/sync_server/sync_server.py +++ b/openpype/modules/sync_server/sync_server.py @@ -404,7 +404,7 @@ async def sync_loop(self): exc_info=True) except Exception: self.stop() - log.warning("Unhandled exception in sync loop, stopping server", + log.warning("Unhandled except. in sync loop, stopping server", exc_info=True) def stop(self): diff --git a/openpype/modules/sync_server/tray/models.py b/openpype/modules/sync_server/tray/models.py index bfe4db32f36..8fdd9487a47 100644 --- a/openpype/modules/sync_server/tray/models.py +++ b/openpype/modules/sync_server/tray/models.py @@ -196,7 +196,7 @@ def sort(self, index, order): break # add default one self.sort['_id'] = 1 - + self.query = self.get_query() # import json # log.debug(json.dumps(self.query, indent=4).\ From 1ceed8169fa90570e660a3be5aaa93cbe0570486 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Apr 2021 19:19:10 +0200 Subject: [PATCH 7/7] SyncServer - handle if globally enabled, but no projects --- .../modules/sync_server/sync_server_module.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index ca4033b3857..a434af9feaa 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -409,6 +409,16 @@ def reset_timer(self): """ self.sync_server_thread.reset_timer() + def get_enabled_projects(self): + """Returns list of projects which have SyncServer enabled.""" + enabled_projects = [] + for project in self.connection.projects(): + project_name = project["name"] + project_settings = self.get_sync_project_setting(project_name) + if project_settings: + enabled_projects.append(project_name) + + return enabled_projects """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): @@ -421,7 +431,7 @@ def get_local_file_path(self, collection, site_name, file_path): return local_file_path def _get_remote_sites_from_settings(self, sync_settings): - if not self.enabled or not sync_settings['enabled']: + if not self.enabled or not sync_settings.get('enabled'): return [] remote_sites = [self.DEFAULT_SITE, self.LOCAL_SITE] @@ -432,7 +442,7 @@ def _get_remote_sites_from_settings(self, sync_settings): def _get_enabled_sites_from_settings(self, sync_settings): sites = [self.DEFAULT_SITE] - if self.enabled and sync_settings['enabled']: + if self.enabled and sync_settings.get('enabled'): sites.append(self.LOCAL_SITE) return sites @@ -453,6 +463,11 @@ def tray_init(self): if not self.enabled: return + enabled_projects = self.get_enabled_projects() + if not enabled_projects: + self.enabled = False + return + self.lock = threading.Lock() try: