From 9c2c1118ac40d6f8ec3b691a947c2a985439da83 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 15:07:47 +0200 Subject: [PATCH 1/7] added notes into client directory --- openpype/client/notes.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 openpype/client/notes.md diff --git a/openpype/client/notes.md b/openpype/client/notes.md new file mode 100644 index 00000000000..a261b86eca9 --- /dev/null +++ b/openpype/client/notes.md @@ -0,0 +1,39 @@ +# Client functionality +## Reason +Preparation for OpenPype v4 server. Goal is to remove direct mongo calls in code to prepare a little bit for different source of data for code before. To start think about database calls less as mongo calls but more universally. To do so was implemented simple wrapper around database calls to not use pymongo specific code. + +Current goal is not to make universal database model which can be easily replaced with any different source of data but to make it close as possible. Current implementation of OpenPype is too tighly connected to pymongo and it's abilities so we're trying to get closer with long term changes that can be used even in current state. + +## Queries +Query functions don't use full potential of mongo queries like very specific queries based on subdictionaries or unknown structures. We try to avoid these calls as much as possible because they'll probably won't be available in future. If it's really necessary a new function can be added but only if it's reasonable for overall logic. All query functions were moved to `~/client/entities.py`. Each function has arguments with available filters and possible reduce of returned keys for each entity. + +## Changes +Changes are a little bit complicated. Mongo has many options how update can happen which had to be reduced also it would be at this stage complicated to validate values which are created or updated thus automation is at this point almost none. Changes can be made using operations available in `~/client/operations.py`. Each operation require project name and entity type, but may require operation specific data. + +### Create +Create operations expect already prepared document data, for that are prepared functions creating skeletal structures of documents (do not fill all required data), except `_id` all data should be right. Existence of entity is not validated so if the same creation operation is send n times it will create the entity n times which can cause issues. + +### Update +Update operation require entity id and keys that should be changed, update dictionary must have {"key": value}. If value should be set in nested dictionary the key must have also all subkeys joined with dot `.` (e.g. `{"data": {"fps": 25}}` -> `{"data.fps": 25}`). To simplify update dictionaries were prepared functions which does that for you, their name has template `prepare__update_data` - they work on comparison of previous document and new document. If there is missing function for requested entity type it is because we didn't need it yet and require implementaion. + +### Delete +Delete operation need entity id. Entity will be deleted from mongo. + + +## What (probably) won't be replaced +Some parts of code are still using direct mongo calls. In most of cases it is for very specific calls that are module specific or their usage will completely change in future. +- Mongo calls that are not project specific (out of `avalon` collection) will be removed or will have to use different mechanism how the data are stored. At this moment it is related to OpenPype settings and logs, ftrack server events, some other data. +- Sync server queries. They're complex and very specific for sync server module. Their replacement will require specific calls to OpenPype server in v4 thus their abstraction with wrapper is irrelevant and would complicate production in v3. +- Project managers (ftrack, kitsu, shotgrid, embedded Project Manager, etc.). Project managers are creating, updating or removing assets in v3, but in v4 will create folders with different structure. Wrapping creation of assets would not help to prepare for v4 because of new data structures. The same can be said about editorial Extract Hierarchy Avalon plugin which create project structure. +- Code parts that is marked as deprecated in v3 or will be deprecated in v4. + - integrate asset legacy publish plugin - already is legacy kept for safety + - integrate thumbnail - thumbnails will be stored in different way in v4 + - input links - link will be stored in different way and will have different mechanism of linking. In v3 are links limited to same entity type "asset <-> asset" or "representation <-> representation". + +## Known missing replacements +- change subset group in loader tool +- integrate subset group +- query input links in openpype lib +- create project in openpype lib +- save/create workfile doc in openpype lib +- integrate hero version From 0d8ab1c17ea388b3f639d1bedab67359463af44c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 15:35:35 +0200 Subject: [PATCH 2/7] fix unsetting of value --- openpype/client/operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/client/operations.py b/openpype/client/operations.py index c4b95bf6962..618cdf9d1e8 100644 --- a/openpype/client/operations.py +++ b/openpype/client/operations.py @@ -444,7 +444,7 @@ def to_mongo_operation(self): set_data = {} for key, value in self._update_data.items(): if value is REMOVED_VALUE: - unset_data[key] = value + unset_data[key] = None else: set_data[key] = value From 0f114331ec45dacc571ab2dc3b04dc869d833d12 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 15:35:57 +0200 Subject: [PATCH 3/7] use client options to change subset group --- openpype/tools/loader/widgets.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/openpype/tools/loader/widgets.py b/openpype/tools/loader/widgets.py index 597c35e89bc..cbf5720803e 100644 --- a/openpype/tools/loader/widgets.py +++ b/openpype/tools/loader/widgets.py @@ -17,6 +17,7 @@ get_thumbnail_id_from_source, get_thumbnail, ) +from openpype.client.operations import OperationsSession, REMOVED_VALUE from openpype.pipeline import HeroVersionType, Anatomy from openpype.pipeline.thumbnail import get_thumbnail_binary from openpype.pipeline.load import ( @@ -614,26 +615,30 @@ def on_context_menu(self, point): box.show() def group_subsets(self, name, asset_ids, items): - field = "data.subsetGroup" + subset_ids = { + item["_id"] + for item in items + if item.get("_id") + } + if not subset_ids: + return if name: - update = {"$set": {field: name}} self.echo("Group subsets to '%s'.." % name) else: - update = {"$unset": {field: ""}} self.echo("Ungroup subsets..") - subsets = list() - for item in items: - subsets.append(item["subset"]) + project_name = self.dbcon.active_project() + op_session = OperationsSession() + for subset_id in subset_ids: + op_session.update_entity( + project_name, + "subset", + subset_id, + {"data.subsetGroup": name or REMOVED_VALUE} + ) - for asset_id in asset_ids: - filtr = { - "type": "subset", - "parent": asset_id, - "name": {"$in": subsets}, - } - self.dbcon.update_many(filtr, update) + op_session.commit() def echo(self, message): print(message) From 7f234e1d814a92cbe1e446aeef7c71a2a2165163 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 16:20:54 +0200 Subject: [PATCH 4/7] fix iterator index acces --- openpype/tools/loader/model.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/tools/loader/model.py b/openpype/tools/loader/model.py index 3ce44ea6c8b..4f1f37b2179 100644 --- a/openpype/tools/loader/model.py +++ b/openpype/tools/loader/model.py @@ -272,11 +272,13 @@ def setData(self, index, value, role=QtCore.Qt.EditRole): # update availability on active site when version changes if self.sync_server.enabled and version_doc: - repre_info = self.sync_server.get_repre_info_for_versions( - project_name, - [version_doc["_id"]], - self.active_site, - self.remote_site + repre_info = list( + self.sync_server.get_repre_info_for_versions( + project_name, + [version_doc["_id"]], + self.active_site, + self.remote_site + ) ) if repre_info: version_doc["data"].update( From 4062bf56f32a6d5ec3e6bf6ec62c61910b3365bd Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 16:36:24 +0200 Subject: [PATCH 5/7] print traceback on crashed dynamic thread --- openpype/tools/utils/lib.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openpype/tools/utils/lib.py b/openpype/tools/utils/lib.py index 99d8c75ab4c..fb2348518a6 100644 --- a/openpype/tools/utils/lib.py +++ b/openpype/tools/utils/lib.py @@ -2,6 +2,7 @@ import sys import contextlib import collections +import traceback from Qt import QtWidgets, QtCore, QtGui import qtawesome @@ -643,7 +644,11 @@ def run(self): def create_qthread(func, *args, **kwargs): class Thread(QtCore.QThread): def run(self): - func(*args, **kwargs) + try: + func(*args, **kwargs) + except: + traceback.print_exception(*sys.exc_info()) + raise return Thread() From fce4e6e3d8f5c3d7f2b929a0328324ce1d951e9b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 16:38:45 +0200 Subject: [PATCH 6/7] fix version specific repre info in loader --- openpype/tools/loader/model.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/openpype/tools/loader/model.py b/openpype/tools/loader/model.py index 4f1f37b2179..929e4978909 100644 --- a/openpype/tools/loader/model.py +++ b/openpype/tools/loader/model.py @@ -272,7 +272,7 @@ def setData(self, index, value, role=QtCore.Qt.EditRole): # update availability on active site when version changes if self.sync_server.enabled and version_doc: - repre_info = list( + repres_info = list( self.sync_server.get_repre_info_for_versions( project_name, [version_doc["_id"]], @@ -280,9 +280,9 @@ def setData(self, index, value, role=QtCore.Qt.EditRole): self.remote_site ) ) - if repre_info: + if repres_info: version_doc["data"].update( - self._get_repre_dict(repre_info[0])) + self._get_repre_dict(repres_info[0])) self.set_version(index, version_doc) @@ -474,29 +474,34 @@ def _fetch(self): last_versions_by_subset_id[subset_id] = hero_version - repre_info = {} + repre_info_by_version_id = {} if self.sync_server.enabled: - version_ids = set() + versions_by_id = {} for _subset_id, doc in last_versions_by_subset_id.items(): - version_ids.add(doc["_id"]) + versions_by_id[doc["_id"]] = doc - repres = self.sync_server.get_repre_info_for_versions( + repres_info = self.sync_server.get_repre_info_for_versions( project_name, - list(version_ids), self.active_site, self.remote_site + list(versions_by_id.keys()), + self.active_site, + self.remote_site ) - for repre in repres: + for repre_info in repres_info: if self._doc_fetching_stop: return + + version_id = repre_info["_id"] + doc = versions_by_id[version_id] doc["active_provider"] = self.active_provider doc["remote_provider"] = self.remote_provider - repre_info[repre["_id"]] = repre + repre_info_by_version_id[version_id] = repre_info self._doc_payload = { "asset_docs_by_id": asset_docs_by_id, "subset_docs_by_id": subset_docs_by_id, "subset_families": subset_families, "last_versions_by_subset_id": last_versions_by_subset_id, - "repre_info_by_version_id": repre_info + "repre_info_by_version_id": repre_info_by_version_id } self.doc_fetched.emit() From c393105e2502ed4b9dba95aa2a49cefbd849c2a0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 23 Aug 2022 16:39:31 +0200 Subject: [PATCH 7/7] use BaseException --- openpype/tools/utils/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/tools/utils/lib.py b/openpype/tools/utils/lib.py index fb2348518a6..97b680b77ee 100644 --- a/openpype/tools/utils/lib.py +++ b/openpype/tools/utils/lib.py @@ -646,7 +646,7 @@ class Thread(QtCore.QThread): def run(self): try: func(*args, **kwargs) - except: + except BaseException: traceback.print_exception(*sys.exc_info()) raise return Thread()