diff --git a/openpype/client/__init__.py b/openpype/client/__init__.py index 16b1dcf3219..e3b4ef51329 100644 --- a/openpype/client/__init__.py +++ b/openpype/client/__init__.py @@ -5,6 +5,7 @@ get_asset_by_id, get_asset_by_name, get_assets, + get_archived_assets, get_asset_ids_with_subsets, get_subset_by_id, @@ -41,6 +42,7 @@ "get_asset_by_id", "get_asset_by_name", "get_assets", + "get_archived_assets", "get_asset_ids_with_subsets", "get_subset_by_id", diff --git a/openpype/client/entities.py b/openpype/client/entities.py index cc4032712c6..4b4a3729fe1 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -139,8 +139,16 @@ def get_asset_by_name(project_name, asset_name, fields=None): return conn.find_one(query_filter, _prepare_fields(fields)) -def get_assets( - project_name, asset_ids=None, asset_names=None, archived=False, fields=None +# NOTE this could be just public function? +# - any better variable name instead of 'standard'? +# - same approach can be used for rest of types +def _get_assets( + project_name, + asset_ids=None, + asset_names=None, + standard=True, + archived=False, + fields=None ): """Assets for specified project by passed filters. @@ -153,6 +161,8 @@ def get_assets( project_name (str): Name of project where to look for queried entities. asset_ids (list[str|ObjectId]): Asset ids that should be found. asset_names (list[str]): Name assets that should be found. + standard (bool): Query standart assets (type 'asset'). + archived (bool): Query archived assets (type 'archived_asset'). fields (list[str]): Fields that should be returned. All fields are returned if 'None' is passed. @@ -161,10 +171,15 @@ def get_assets( passed filters. """ - asset_types = ["asset"] + asset_types = [] + if standard: + asset_types.append("asset") if archived: asset_types.append("archived_asset") + if not asset_types: + return [] + if len(asset_types) == 1: query_filter = {"type": asset_types[0]} else: @@ -186,6 +201,68 @@ def get_assets( return conn.find(query_filter, _prepare_fields(fields)) +def get_assets( + project_name, + asset_ids=None, + asset_names=None, + archived=False, + fields=None +): + """Assets for specified project by passed filters. + + Passed filters (ids and names) are always combined so all conditions must + match. + + To receive all assets from project just keep filters empty. + + Args: + project_name (str): Name of project where to look for queried entities. + asset_ids (list[str|ObjectId]): Asset ids that should be found. + asset_names (list[str]): Name assets that should be found. + archived (bool): Add also archived assets. + fields (list[str]): Fields that should be returned. All fields are + returned if 'None' is passed. + + Returns: + Cursor: Query cursor as iterable which returns asset documents matching + passed filters. + """ + + return _get_assets( + project_name, asset_ids, asset_names, True, archived, fields + ) + + +def get_archived_assets( + project_name, + asset_ids=None, + asset_names=None, + fields=None +): + """Archived assets for specified project by passed filters. + + Passed filters (ids and names) are always combined so all conditions must + match. + + To receive all archived assets from project just keep filters empty. + + Args: + project_name (str): Name of project where to look for queried entities. + asset_ids (list[str|ObjectId]): Asset ids that should be found. + asset_names (list[str]): Name assets that should be found. + fields (list[str]): Fields that should be returned. All fields are + returned if 'None' is passed. + + Returns: + Cursor: Query cursor as iterable which returns asset documents matching + passed filters. + """ + + return _get_assets( + project_name, asset_ids, asset_names, False, True, fields + ) + + def get_asset_ids_with_subsets(project_name, asset_ids=None): """Find out which assets have existing subsets. @@ -432,6 +509,7 @@ def _get_versions( project_name, subset_ids=None, version_ids=None, + versions=None, standard=True, hero=False, fields=None @@ -462,6 +540,16 @@ def _get_versions( return [] query_filter["_id"] = {"$in": version_ids} + if versions is not None: + versions = list(versions) + if not versions: + return [] + + if len(versions) == 1: + query_filter["name"] = versions[0] + else: + query_filter["name"] = {"$in": versions} + conn = _get_project_connection(project_name) return conn.find(query_filter, _prepare_fields(fields)) @@ -471,6 +559,7 @@ def get_versions( project_name, version_ids=None, subset_ids=None, + versions=None, hero=False, fields=None ): @@ -484,6 +573,8 @@ def get_versions( Filter ignored if 'None' is passed. subset_ids (list[str]): Subset ids that will be queried. Filter ignored if 'None' is passed. + versions (list[int]): Version names (as integers). + Filter ignored if 'None' is passed. hero (bool): Look also for hero versions. fields (list[str]): Fields that should be returned. All fields are returned if 'None' is passed. @@ -496,6 +587,7 @@ def get_versions( project_name, subset_ids, version_ids, + versions, standard=True, hero=hero, fields=fields diff --git a/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py b/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py index 975e49cb289..361aa98d160 100644 --- a/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py +++ b/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py @@ -1,8 +1,8 @@ import json +from openpype.client import get_project from openpype.api import ProjectSettings from openpype.lib import create_project -from openpype.pipeline import AvalonMongoDB from openpype.settings import SaveWarningExc from openpype_modules.ftrack.lib import ( @@ -363,12 +363,8 @@ def launch(self, session, entities, event): project_name = project_entity["full_name"] # Try to find project document - dbcon = AvalonMongoDB() - dbcon.install() - dbcon.Session["AVALON_PROJECT"] = project_name - project_doc = dbcon.find_one({ - "type": "project" - }) + project_doc = get_project(project_name) + # Create project if is not available # - creation is required to be able set project anatomy and attributes if not project_doc: @@ -376,9 +372,7 @@ def launch(self, session, entities, event): self.log.info("Creating project \"{} [{}]\"".format( project_name, project_code )) - create_project(project_name, project_code, dbcon=dbcon) - - dbcon.uninstall() + create_project(project_name, project_code) project_settings = ProjectSettings(project_name) project_anatomy_settings = project_settings["project_anatomy"] diff --git a/openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index b5f199b3e4a..a4e791aaf0f 100644 --- a/openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -12,6 +12,12 @@ import arrow import ftrack_api +from openpype.client import ( + get_project, + get_assets, + get_archived_assets, + get_asset_ids_with_subsets +) from openpype.pipeline import AvalonMongoDB, schema from openpype_modules.ftrack.lib import ( @@ -149,12 +155,11 @@ def cust_attr_types_by_id(self): @property def avalon_entities(self): if self._avalon_ents is None: + project_name = self.cur_project["full_name"] self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = ( - self.cur_project["full_name"] - ) - avalon_project = self.dbcon.find_one({"type": "project"}) - avalon_entities = list(self.dbcon.find({"type": "asset"})) + self.dbcon.Session["AVALON_PROJECT"] = project_name + avalon_project = get_project(project_name) + avalon_entities = list(get_assets(project_name)) self._avalon_ents = (avalon_project, avalon_entities) return self._avalon_ents @@ -284,28 +289,21 @@ def handle_missing_ftrack_id(self, doc): self._avalon_ents_by_ftrack_id[ftrack_id] = doc @property - def avalon_subsets_by_parents(self): - if self._avalon_subsets_by_parents is None: - self._avalon_subsets_by_parents = collections.defaultdict(list) - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = ( - self.cur_project["full_name"] + def avalon_asset_ids_with_subsets(self): + if self._avalon_asset_ids_with_subsets is None: + project_name = self.cur_project["full_name"] + self._avalon_asset_ids_with_subsets = get_asset_ids_with_subsets( + project_name ) - for subset in self.dbcon.find({"type": "subset"}): - self._avalon_subsets_by_parents[subset["parent"]].append( - subset - ) - return self._avalon_subsets_by_parents + + return self._avalon_asset_ids_with_subsets @property def avalon_archived_by_id(self): if self._avalon_archived_by_id is None: self._avalon_archived_by_id = {} - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = ( - self.cur_project["full_name"] - ) - for asset in self.dbcon.find({"type": "archived_asset"}): + project_name = self.cur_project["full_name"] + for asset in get_archived_assets(project_name): self._avalon_archived_by_id[asset["_id"]] = asset return self._avalon_archived_by_id @@ -327,7 +325,7 @@ def changeability_by_mongo_id(self): avalon_project, avalon_entities = self.avalon_entities self._changeability_by_mongo_id[avalon_project["_id"]] = False self._bubble_changeability( - list(self.avalon_subsets_by_parents.keys()) + list(self.avalon_asset_ids_with_subsets) ) return self._changeability_by_mongo_id @@ -449,14 +447,9 @@ def _bubble_changeability(self, unchangeable_ids): if not entity: # if entity is not found then it is subset without parent if entity_id in unchangeable_ids: - _subset_ids = [ - str(sub["_id"]) for sub in - self.avalon_subsets_by_parents[entity_id] - ] - joined_subset_ids = "| ".join(_subset_ids) self.log.warning(( - "Parent <{}> for subsets <{}> does not exist" - ).format(str(entity_id), joined_subset_ids)) + "Parent <{}> with subsets does not exist" + ).format(str(entity_id))) else: self.log.warning(( "In avalon are entities without valid parents that" @@ -483,7 +476,7 @@ def reset_variables(self): self._avalon_ents_by_parent_id = None self._avalon_ents_by_ftrack_id = None self._avalon_ents_by_name = None - self._avalon_subsets_by_parents = None + self._avalon_asset_ids_with_subsets = None self._changeability_by_mongo_id = None self._avalon_archived_by_id = None self._avalon_archived_by_name = None diff --git a/openpype/modules/ftrack/event_handlers_server/event_user_assigment.py b/openpype/modules/ftrack/event_handlers_server/event_user_assigment.py index 593fc5e5968..82b79e986b8 100644 --- a/openpype/modules/ftrack/event_handlers_server/event_user_assigment.py +++ b/openpype/modules/ftrack/event_handlers_server/event_user_assigment.py @@ -1,11 +1,9 @@ import re import subprocess +from openpype.client import get_asset_by_id, get_asset_by_name from openpype_modules.ftrack.lib import BaseEvent from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY -from openpype.pipeline import AvalonMongoDB - -from bson.objectid import ObjectId from openpype.api import Anatomy, get_project_settings @@ -36,8 +34,6 @@ class UserAssigmentEvent(BaseEvent): 3) path to publish files of task user was (de)assigned to """ - db_con = AvalonMongoDB() - def error(self, *err): for e in err: self.log.error(e) @@ -101,26 +97,16 @@ def _get_asset(self, task): :rtype: dict """ parent = task['parent'] - self.db_con.install() - self.db_con.Session['AVALON_PROJECT'] = task['project']['full_name'] - + project_name = task["project"]["full_name"] avalon_entity = None parent_id = parent['custom_attributes'].get(CUST_ATTR_ID_KEY) if parent_id: - parent_id = ObjectId(parent_id) - avalon_entity = self.db_con.find_one({ - '_id': parent_id, - 'type': 'asset' - }) + avalon_entity = get_asset_by_id(project_name, parent_id) if not avalon_entity: - avalon_entity = self.db_con.find_one({ - 'type': 'asset', - 'name': parent['name'] - }) + avalon_entity = get_asset_by_name(project_name, parent["name"]) if not avalon_entity: - self.db_con.uninstall() msg = 'Entity "{}" not found in avalon database'.format( parent['name'] ) @@ -129,7 +115,6 @@ def _get_asset(self, task): 'success': False, 'message': msg } - self.db_con.uninstall() return avalon_entity def _get_hierarchy(self, asset): diff --git a/openpype/modules/ftrack/event_handlers_user/action_applications.py b/openpype/modules/ftrack/event_handlers_user/action_applications.py index b25bc1b5cb6..102f04c9566 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_applications.py +++ b/openpype/modules/ftrack/event_handlers_user/action_applications.py @@ -1,5 +1,6 @@ import os +from openpype.client import get_project from openpype_modules.ftrack.lib import BaseAction from openpype.lib.applications import ( ApplicationManager, @@ -7,7 +8,6 @@ ApplictionExecutableNotFound, CUSTOM_LAUNCH_APP_GROUPS ) -from openpype.pipeline import AvalonMongoDB class AppplicationsAction(BaseAction): @@ -25,7 +25,6 @@ def __init__(self, *args, **kwargs): super(AppplicationsAction, self).__init__(*args, **kwargs) self.application_manager = ApplicationManager() - self.dbcon = AvalonMongoDB() @property def discover_identifier(self): @@ -110,12 +109,7 @@ def discover(self, session, entities, event): if avalon_project_doc is None: ft_project = self.get_project_from_entity(entity) project_name = ft_project["full_name"] - if not self.dbcon.is_installed(): - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = project_name - avalon_project_doc = self.dbcon.find_one({ - "type": "project" - }) or False + avalon_project_doc = get_project(project_name) or False event["data"]["avalon_project_doc"] = avalon_project_doc if not avalon_project_doc: diff --git a/openpype/modules/ftrack/event_handlers_user/action_delete_asset.py b/openpype/modules/ftrack/event_handlers_user/action_delete_asset.py index ee5c3d0d97d..03d029b0c18 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delete_asset.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delete_asset.py @@ -4,6 +4,7 @@ from bson.objectid import ObjectId +from openpype.client import get_assets, get_subsets from openpype.pipeline import AvalonMongoDB from openpype_modules.ftrack.lib import BaseAction, statics_icon from openpype_modules.ftrack.lib.avalon_sync import create_chunks @@ -91,10 +92,8 @@ def interface(self, session, entities, event): continue ftrack_id = entity.get("entityId") - if not ftrack_id: - continue - - ftrack_ids.append(ftrack_id) + if ftrack_id: + ftrack_ids.append(ftrack_id) if project_in_selection: msg = "It is not possible to use this action on project entity." @@ -120,48 +119,51 @@ def interface(self, session, entities, event): "message": "Invalid selection for this action (Bug)" } - if entities[0].entity_type.lower() == "project": - project = entities[0] - else: - project = entities[0]["project"] - + project = self.get_project_from_entity(entities[0], session) project_name = project["full_name"] self.dbcon.Session["AVALON_PROJECT"] = project_name - selected_av_entities = list(self.dbcon.find({ - "type": "asset", - "data.ftrackId": {"$in": ftrack_ids} - })) + asset_docs = list(get_assets( + project_name, + fields=["_id", "name", "data.ftrackId", "data.parents"] + )) + selected_av_entities = [] + found_ftrack_ids = set() + asset_docs_by_name = collections.defaultdict(list) + for asset_doc in asset_docs: + ftrack_id = asset_doc["data"].get("ftrackId") + if ftrack_id: + found_ftrack_ids.add(ftrack_id) + if ftrack_id in entity_mapping: + selected_av_entities.append(asset_doc) + + asset_name = asset_doc["name"] + asset_docs_by_name[asset_name].append(asset_doc) + found_without_ftrack_id = {} - if len(selected_av_entities) != len(ftrack_ids): - found_ftrack_ids = [ - ent["data"]["ftrackId"] for ent in selected_av_entities - ] - for ftrack_id, entity in entity_mapping.items(): - if ftrack_id in found_ftrack_ids: - continue + for ftrack_id, entity in entity_mapping.items(): + if ftrack_id in found_ftrack_ids: + continue - av_ents_by_name = list(self.dbcon.find({ - "type": "asset", - "name": entity["name"] - })) - if not av_ents_by_name: - continue + av_ents_by_name = asset_docs_by_name[entity["name"]] + if not av_ents_by_name: + continue - ent_path_items = [ent["name"] for ent in entity["link"]] - parents = ent_path_items[1:len(ent_path_items)-1:] - # TODO we should say to user that - # few of them are missing in avalon - for av_ent in av_ents_by_name: - if av_ent["data"]["parents"] != parents: - continue + ent_path_items = [ent["name"] for ent in entity["link"]] + end_index = len(ent_path_items) - 1 + parents = ent_path_items[1:end_index:] + # TODO we should say to user that + # few of them are missing in avalon + for av_ent in av_ents_by_name: + if av_ent["data"]["parents"] != parents: + continue - # TODO we should say to user that found entity - # with same name does not match same ftrack id? - if "ftrackId" not in av_ent["data"]: - selected_av_entities.append(av_ent) - found_without_ftrack_id[str(av_ent["_id"])] = ftrack_id - break + # TODO we should say to user that found entity + # with same name does not match same ftrack id? + if "ftrackId" not in av_ent["data"]: + selected_av_entities.append(av_ent) + found_without_ftrack_id[str(av_ent["_id"])] = ftrack_id + break if not selected_av_entities: return { @@ -206,10 +208,7 @@ def interface(self, session, entities, event): items.append(id_item) asset_ids = [ent["_id"] for ent in selected_av_entities] - subsets_for_selection = self.dbcon.find({ - "type": "subset", - "parent": {"$in": asset_ids} - }) + subsets_for_selection = get_subsets(project_name, asset_ids=asset_ids) asset_ending = "" if len(selected_av_entities) > 1: @@ -459,13 +458,9 @@ def launch(self, session, entities, event): if len(assets_to_delete) > 0: map_av_ftrack_id = spec_data["without_ftrack_id"] # Prepare data when deleting whole avalon asset - avalon_assets = self.dbcon.find( - {"type": "asset"}, - { - "_id": 1, - "data.visualParent": 1, - "data.ftrackId": 1 - } + avalon_assets = get_assets( + project_name, + fields=["_id", "data.visualParent", "data.ftrackId"] ) avalon_assets_by_parent = collections.defaultdict(list) for asset in avalon_assets: diff --git a/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py index a0bf6622e9e..3400c509ab0 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py @@ -5,7 +5,12 @@ import clique from pymongo import UpdateOne - +from openpype.client import ( + get_assets, + get_subsets, + get_versions, + get_representations +) from openpype.api import Anatomy from openpype.lib import StringTemplate, TemplateUnsolved from openpype.pipeline import AvalonMongoDB @@ -198,10 +203,9 @@ def launch(self, session, entities, event): self.log.debug("Project is set to {}".format(project_name)) # Get Assets from avalon database - assets = list(self.dbcon.find({ - "type": "asset", - "name": {"$in": avalon_asset_names} - })) + assets = list( + get_assets(project_name, asset_names=avalon_asset_names) + ) asset_id_to_name_map = { asset["_id"]: asset["name"] for asset in assets } @@ -210,10 +214,9 @@ def launch(self, session, entities, event): self.log.debug("Collected assets ({})".format(len(asset_ids))) # Get Subsets - subsets = list(self.dbcon.find({ - "type": "subset", - "parent": {"$in": asset_ids} - })) + subsets = list( + get_subsets(project_name, asset_ids=asset_ids) + ) subsets_by_id = {} subset_ids = [] for subset in subsets: @@ -230,10 +233,9 @@ def launch(self, session, entities, event): self.log.debug("Collected subsets ({})".format(len(subset_ids))) # Get Versions - versions = list(self.dbcon.find({ - "type": "version", - "parent": {"$in": subset_ids} - })) + versions = list( + get_versions(project_name, subset_ids=subset_ids) + ) versions_by_parent = collections.defaultdict(list) for ent in versions: @@ -295,10 +297,9 @@ def sort_func(ent): "message": msg } - repres = list(self.dbcon.find({ - "type": "representation", - "parent": {"$in": version_ids} - })) + repres = list( + get_representations(project_name, version_ids=version_ids) + ) self.log.debug( "Collected representations to remove ({})".format(len(repres)) diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py index 86d88ef7cce..4b799b092bc 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py @@ -3,8 +3,13 @@ import json import collections -from bson.objectid import ObjectId - +from openpype.client import ( + get_project, + get_assets, + get_subsets, + get_versions, + get_representations +) from openpype.api import Anatomy, config from openpype_modules.ftrack.lib import BaseAction, statics_icon from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY @@ -18,11 +23,9 @@ process_single_file, process_sequence ) -from openpype.pipeline import AvalonMongoDB class Delivery(BaseAction): - identifier = "delivery.action" label = "Delivery" description = "Deliver data to client" @@ -30,11 +33,6 @@ class Delivery(BaseAction): icon = statics_icon("ftrack", "action_icons", "Delivery.svg") settings_key = "delivery_action" - def __init__(self, *args, **kwargs): - self.dbcon = AvalonMongoDB() - - super(Delivery, self).__init__(*args, **kwargs) - def discover(self, session, entities, event): is_valid = False for entity in entities: @@ -57,9 +55,7 @@ def interface(self, session, entities, event): project_entity = self.get_project_from_entity(entities[0]) project_name = project_entity["full_name"] - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = project_name - project_doc = self.dbcon.find_one({"type": "project"}, {"name": True}) + project_doc = get_project(project_name, fields=["name"]) if not project_doc: return { "success": False, @@ -68,8 +64,7 @@ def interface(self, session, entities, event): ).format(project_name) } - repre_names = self._get_repre_names(session, entities) - self.dbcon.uninstall() + repre_names = self._get_repre_names(project_name, session, entities) items.append({ "type": "hidden", @@ -198,17 +193,21 @@ def interface(self, session, entities, event): "title": title } - def _get_repre_names(self, session, entities): - version_ids = self._get_interest_version_ids(session, entities) + def _get_repre_names(self, project_name, session, entities): + version_ids = self._get_interest_version_ids( + project_name, session, entities + ) if not version_ids: return [] - repre_docs = self.dbcon.find({ - "type": "representation", - "parent": {"$in": version_ids} - }) - return list(sorted(repre_docs.distinct("name"))) + repre_docs = get_representations( + project_name, + version_ids=version_ids, + fields=["name"] + ) + repre_names = {repre_doc["name"] for repre_doc in repre_docs} + return list(sorted(repre_names)) - def _get_interest_version_ids(self, session, entities): + def _get_interest_version_ids(self, project_name, session, entities): # Extract AssetVersion entities asset_versions = self._extract_asset_versions(session, entities) # Prepare Asset ids @@ -235,14 +234,18 @@ def _get_interest_version_ids(self, session, entities): subset_names.add(asset["name"]) version_nums.add(asset_version["version"]) - asset_docs_by_ftrack_id = self._get_asset_docs(session, parent_ids) + asset_docs_by_ftrack_id = self._get_asset_docs( + project_name, session, parent_ids + ) subset_docs = self._get_subset_docs( + project_name, asset_docs_by_ftrack_id, subset_names, asset_versions, assets_by_id ) version_docs = self._get_version_docs( + project_name, asset_docs_by_ftrack_id, subset_docs, version_nums, @@ -290,6 +293,7 @@ def _get_asset_version_ids_from_review_sessions( def _get_version_docs( self, + project_name, asset_docs_by_ftrack_id, subset_docs, version_nums, @@ -300,11 +304,11 @@ def _get_version_docs( subset_doc["_id"]: subset_doc for subset_doc in subset_docs } - version_docs = list(self.dbcon.find({ - "type": "version", - "parent": {"$in": list(subset_docs_by_id.keys())}, - "name": {"$in": list(version_nums)} - })) + version_docs = list(get_versions( + project_name, + subset_ids=subset_docs_by_id.keys(), + versions=version_nums + )) version_docs_by_parent_id = collections.defaultdict(dict) for version_doc in version_docs: subset_doc = subset_docs_by_id[version_doc["parent"]] @@ -345,6 +349,7 @@ def _get_version_docs( def _get_subset_docs( self, + project_name, asset_docs_by_ftrack_id, subset_names, asset_versions, @@ -354,11 +359,11 @@ def _get_subset_docs( asset_doc["_id"] for asset_doc in asset_docs_by_ftrack_id.values() ] - subset_docs = list(self.dbcon.find({ - "type": "subset", - "parent": {"$in": asset_doc_ids}, - "name": {"$in": list(subset_names)} - })) + subset_docs = list(get_subsets( + project_name, + asset_ids=asset_doc_ids, + subset_names=subset_names + )) subset_docs_by_parent_id = collections.defaultdict(dict) for subset_doc in subset_docs: asset_id = subset_doc["parent"] @@ -385,15 +390,21 @@ def _get_subset_docs( filtered_subsets.append(subset_doc) return filtered_subsets - def _get_asset_docs(self, session, parent_ids): - asset_docs = list(self.dbcon.find({ - "type": "asset", - "data.ftrackId": {"$in": list(parent_ids)} - })) + def _get_asset_docs(self, project_name, session, parent_ids): + asset_docs = list(get_assets( + project_name, fields=["_id", "name", "data.ftrackId"] + )) + asset_docs_by_id = {} + asset_docs_by_name = {} asset_docs_by_ftrack_id = {} for asset_doc in asset_docs: + asset_id = str(asset_doc["_id"]) + asset_name = asset_doc["name"] ftrack_id = asset_doc["data"].get("ftrackId") + + asset_docs_by_id[asset_id] = asset_doc + asset_docs_by_name[asset_name] = asset_doc if ftrack_id: asset_docs_by_ftrack_id[ftrack_id] = asset_doc @@ -406,15 +417,15 @@ def _get_asset_docs(self, session, parent_ids): avalon_mongo_id_values = query_custom_attributes( session, [attr_def["id"]], parent_ids, True ) - entity_ids_by_mongo_id = { - ObjectId(item["value"]): item["entity_id"] - for item in avalon_mongo_id_values - if item["value"] - } - missing_ids = set(parent_ids) - for entity_id in set(entity_ids_by_mongo_id.values()): - if entity_id in missing_ids: + for item in avalon_mongo_id_values: + if not item["value"]: + continue + asset_id = item["value"] + entity_id = item["entity_id"] + asset_doc = asset_docs_by_id.get(asset_id) + if asset_doc: + asset_docs_by_ftrack_id[entity_id] = asset_doc missing_ids.remove(entity_id) entity_ids_by_name = {} @@ -427,36 +438,10 @@ def _get_asset_docs(self, session, parent_ids): for entity in not_found_entities } - expressions = [] - if entity_ids_by_mongo_id: - expression = { - "type": "asset", - "_id": {"$in": list(entity_ids_by_mongo_id.keys())} - } - expressions.append(expression) - - if entity_ids_by_name: - expression = { - "type": "asset", - "name": {"$in": list(entity_ids_by_name.keys())} - } - expressions.append(expression) - - if expressions: - if len(expressions) == 1: - filter = expressions[0] - else: - filter = {"$or": expressions} - - asset_docs = self.dbcon.find(filter) - for asset_doc in asset_docs: - if asset_doc["_id"] in entity_ids_by_mongo_id: - entity_id = entity_ids_by_mongo_id[asset_doc["_id"]] - asset_docs_by_ftrack_id[entity_id] = asset_doc - - elif asset_doc["name"] in entity_ids_by_name: - entity_id = entity_ids_by_name[asset_doc["name"]] - asset_docs_by_ftrack_id[entity_id] = asset_doc + for asset_name, entity_id in entity_ids_by_name.items(): + asset_doc = asset_docs_by_name.get(asset_name) + if asset_doc: + asset_docs_by_ftrack_id[entity_id] = asset_doc return asset_docs_by_ftrack_id @@ -490,7 +475,6 @@ def launch(self, session, entities, event): session.commit() try: - self.dbcon.install() report = self.real_launch(session, entities, event) except Exception as exc: @@ -516,7 +500,6 @@ def launch(self, session, entities, event): else: job["status"] = "failed" session.commit() - self.dbcon.uninstall() if not report["success"]: self.show_interface( @@ -558,16 +541,15 @@ def real_launch(self, session, entities, event): if not os.path.exists(location_path): os.makedirs(location_path) - self.dbcon.Session["AVALON_PROJECT"] = project_name - self.log.debug("Collecting representations to process.") - version_ids = self._get_interest_version_ids(session, entities) - repres_to_deliver = list(self.dbcon.find({ - "type": "representation", - "parent": {"$in": version_ids}, - "name": {"$in": repre_names} - })) - + version_ids = self._get_interest_version_ids( + project_name, session, entities + ) + repres_to_deliver = list(get_representations( + project_name, + representation_names=repre_names, + version_ids=version_ids + )) anatomy = Anatomy(project_name) format_dict = get_format_dict(anatomy, location_path) diff --git a/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py b/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py index c7237a11507..d30c41a7499 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py +++ b/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py @@ -7,6 +7,10 @@ import ftrack_api +from openpype.client import ( + get_project, + get_assets, +) from openpype.api import get_project_settings from openpype.lib import ( get_workfile_template_key, @@ -14,7 +18,6 @@ Anatomy, StringTemplate, ) -from openpype.pipeline import AvalonMongoDB from openpype_modules.ftrack.lib import BaseAction, statics_icon from openpype_modules.ftrack.lib.avalon_sync import create_chunks @@ -248,10 +251,8 @@ def in_job_process( # Find matchin asset documents and map them by ftrack task entities # - result stored to 'asset_docs_with_task_entities' is list with # tuple `(asset document, [task entitis, ...])` - dbcon = AvalonMongoDB() - dbcon.Session["AVALON_PROJECT"] = project_name # Quety all asset documents - asset_docs = list(dbcon.find({"type": "asset"})) + asset_docs = list(get_assets(project_name)) job_entity["data"] = json.dumps({ "description": "(1/3) Asset documents queried." }) @@ -276,7 +277,7 @@ def in_job_process( # Keep placeholders in the template unfilled host_name = "{app}" extension = "{ext}" - project_doc = dbcon.find_one({"type": "project"}) + project_doc = get_project(project_name) project_settings = get_project_settings(project_name) anatomy = Anatomy(project_name) templates_by_key = {} diff --git a/openpype/modules/ftrack/event_handlers_user/action_prepare_project.py b/openpype/modules/ftrack/event_handlers_user/action_prepare_project.py index 0b14e7aa2b3..e9dc11de9f7 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_prepare_project.py +++ b/openpype/modules/ftrack/event_handlers_user/action_prepare_project.py @@ -1,8 +1,8 @@ import json +from openpype.client import get_project from openpype.api import ProjectSettings from openpype.lib import create_project -from openpype.pipeline import AvalonMongoDB from openpype.settings import SaveWarningExc from openpype_modules.ftrack.lib import ( @@ -389,12 +389,8 @@ def launch(self, session, entities, event): project_name = project_entity["full_name"] # Try to find project document - dbcon = AvalonMongoDB() - dbcon.install() - dbcon.Session["AVALON_PROJECT"] = project_name - project_doc = dbcon.find_one({ - "type": "project" - }) + project_doc = get_project(project_name) + # Create project if is not available # - creation is required to be able set project anatomy and attributes if not project_doc: @@ -402,9 +398,7 @@ def launch(self, session, entities, event): self.log.info("Creating project \"{} [{}]\"".format( project_name, project_code )) - create_project(project_name, project_code, dbcon=dbcon) - - dbcon.uninstall() + create_project(project_name, project_code) project_settings = ProjectSettings(project_name) project_anatomy_settings = project_settings["project_anatomy"] diff --git a/openpype/modules/ftrack/event_handlers_user/action_rv.py b/openpype/modules/ftrack/event_handlers_user/action_rv.py index 040ca755829..2480ea7f95a 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_rv.py +++ b/openpype/modules/ftrack/event_handlers_user/action_rv.py @@ -5,9 +5,16 @@ import ftrack_api +from openpype.client import ( + get_asset_by_name, + get_subset_by_name, + get_version_by_name, + get_representation_by_name +) +from openpype.api import Anatomy from openpype.pipeline import ( get_representation_path, - legacy_io, + AvalonMongoDB, ) from openpype_modules.ftrack.lib import BaseAction, statics_icon @@ -255,9 +262,10 @@ def get_file_paths(self, session, event): "Component", list(event["data"]["values"].values())[0] )["version"]["asset"]["parent"]["link"][0] project = session.get(link["type"], link["id"]) - os.environ["AVALON_PROJECT"] = project["name"] - legacy_io.Session["AVALON_PROJECT"] = project["name"] - legacy_io.install() + project_name = project["full_name"] + dbcon = AvalonMongoDB() + dbcon.Session["AVALON_PROJECT"] = project_name + anatomy = Anatomy(project_name) location = ftrack_api.Session().pick_location() @@ -281,37 +289,38 @@ def get_file_paths(self, session, event): if online_source: continue - asset = legacy_io.find_one({"type": "asset", "name": parent_name}) - subset = legacy_io.find_one( - { - "type": "subset", - "name": component["version"]["asset"]["name"], - "parent": asset["_id"] - } + subset_name = component["version"]["asset"]["name"] + version_name = component["version"]["version"] + representation_name = component["file_type"][1:] + + asset_doc = get_asset_by_name( + project_name, parent_name, fields=["_id"] ) - version = legacy_io.find_one( - { - "type": "version", - "name": component["version"]["version"], - "parent": subset["_id"] - } + subset_doc = get_subset_by_name( + project_name, + subset_name=subset_name, + asset_id=asset_doc["_id"] ) - representation = legacy_io.find_one( - { - "type": "representation", - "parent": version["_id"], - "name": component["file_type"][1:] - } + version_doc = get_version_by_name( + project_name, + version=version_name, + subset_id=subset_doc["_id"] ) - if representation is None: - representation = legacy_io.find_one( - { - "type": "representation", - "parent": version["_id"], - "name": "preview" - } + repre_doc = get_representation_by_name( + project_name, + version_id=version_doc["_id"], + representation_name=representation_name + ) + if not repre_doc: + repre_doc = get_representation_by_name( + project_name, + version_id=version_doc["_id"], + representation_name="preview" ) - paths.append(get_representation_path(representation)) + + paths.append(get_representation_path( + repre_doc, root=anatomy.roots, dbcon=dbcon + )) return paths diff --git a/openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py b/openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py index 62fdfa2bddd..d655dddcaf3 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py +++ b/openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py @@ -5,6 +5,14 @@ from bson.objectid import ObjectId +from openpype.client import ( + get_project, + get_asset_by_id, + get_assets, + get_subset_by_name, + get_version_by_name, + get_representations +) from openpype_modules.ftrack.lib import BaseAction, statics_icon from openpype.api import Anatomy from openpype.pipeline import AvalonMongoDB @@ -385,7 +393,7 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con): db_con.Session["AVALON_PROJECT"] = project_name - avalon_project = db_con.find_one({"type": "project"}) + avalon_project = get_project(project_name) output["project"] = avalon_project if not avalon_project: @@ -399,19 +407,17 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con): asset_mongo_id = parent["custom_attributes"].get(CUST_ATTR_ID_KEY) if asset_mongo_id: try: - asset_mongo_id = ObjectId(asset_mongo_id) - asset_ent = db_con.find_one({ - "type": "asset", - "_id": asset_mongo_id - }) + asset_ent = get_asset_by_id(project_name, asset_mongo_id) except Exception: pass if not asset_ent: - asset_ent = db_con.find_one({ - "type": "asset", - "data.ftrackId": parent["id"] - }) + asset_docs = get_assets(project_name, asset_names=[parent["name"]]) + for asset_doc in asset_docs: + ftrack_id = asset_doc.get("data", {}).get("ftrackId") + if ftrack_id == parent["id"]: + asset_ent = asset_doc + break output["asset"] = asset_ent @@ -422,13 +428,11 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con): ) return output - asset_mongo_id = asset_ent["_id"] - - subset_ent = db_con.find_one({ - "type": "subset", - "parent": asset_mongo_id, - "name": subset_name - }) + subset_ent = get_subset_by_name( + project_name, + subset_name=subset_name, + asset_id=asset_ent["_id"] + ) output["subset"] = subset_ent @@ -439,11 +443,11 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con): ).format(subset_name, ent_path) return output - version_ent = db_con.find_one({ - "type": "version", - "name": version, - "parent": subset_ent["_id"] - }) + version_ent = get_version_by_name( + project_name, + version, + subset_ent["_id"] + ) output["version"] = version_ent @@ -454,10 +458,10 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con): ).format(version, subset_name, ent_path) return output - repre_ents = list(db_con.find({ - "type": "representation", - "parent": version_ent["_id"] - })) + repre_ents = list(get_representations( + project_name, + version_ids=[version_ent["_id"]] + )) output["representations"] = repre_ents return output diff --git a/openpype/modules/ftrack/lib/avalon_sync.py b/openpype/modules/ftrack/lib/avalon_sync.py index e4ba651bfdd..68b5c62c53d 100644 --- a/openpype/modules/ftrack/lib/avalon_sync.py +++ b/openpype/modules/ftrack/lib/avalon_sync.py @@ -6,6 +6,14 @@ import six +from openpype.client import ( + get_project, + get_assets, + get_archived_assets, + get_subsets, + get_versions, + get_representations +) from openpype.api import ( Logger, get_anatomy_settings @@ -576,6 +584,10 @@ def launch_setup(self, project_full_name): self.ft_project_id = ft_project_id self.entities_dict = entities_dict + @property + def project_name(self): + return self.entities_dict[self.ft_project_id]["name"] + @property def avalon_ents_by_id(self): """ @@ -660,9 +672,9 @@ def avalon_archived_ents(self): (list) of assets """ if self._avalon_archived_ents is None: - self._avalon_archived_ents = [ - ent for ent in self.dbcon.find({"type": "archived_asset"}) - ] + self._avalon_archived_ents = list( + get_archived_assets(self.project_name) + ) return self._avalon_archived_ents @property @@ -730,7 +742,7 @@ def subsets_by_parent_id(self): """ if self._subsets_by_parent_id is None: self._subsets_by_parent_id = collections.defaultdict(list) - for subset in self.dbcon.find({"type": "subset"}): + for subset in get_subsets(self.project_name): self._subsets_by_parent_id[str(subset["parent"])].append( subset ) @@ -1421,8 +1433,8 @@ def prepare_avalon_entities(self, ft_project_name): # Avalon entities self.dbcon.install() self.dbcon.Session["AVALON_PROJECT"] = ft_project_name - avalon_project = self.dbcon.find_one({"type": "project"}) - avalon_entities = self.dbcon.find({"type": "asset"}) + avalon_project = get_project(ft_project_name) + avalon_entities = get_assets(ft_project_name) self.avalon_project = avalon_project self.avalon_entities = avalon_entities @@ -2258,46 +2270,37 @@ def _bubble_changeability(self, unchangeable_ids): self._delete_subsets_without_asset(subsets_to_remove) def _delete_subsets_without_asset(self, not_existing_parents): - subset_ids = [] - version_ids = [] repre_ids = [] to_delete = [] + subset_ids = [] for parent_id in not_existing_parents: subsets = self.subsets_by_parent_id.get(parent_id) if not subsets: continue for subset in subsets: - if subset.get("type") != "subset": - continue - subset_ids.append(subset["_id"]) + if subset.get("type") == "subset": + subset_ids.append(subset["_id"]) - db_subsets = self.dbcon.find({ - "_id": {"$in": subset_ids}, - "type": "subset" - }) - if not db_subsets: - return - - db_versions = self.dbcon.find({ - "parent": {"$in": subset_ids}, - "type": "version" - }) - if db_versions: - version_ids = [ver["_id"] for ver in db_versions] - - db_repres = self.dbcon.find({ - "parent": {"$in": version_ids}, - "type": "representation" - }) - if db_repres: - repre_ids = [repre["_id"] for repre in db_repres] + db_versions = get_versions( + self.project_name, + subset_ids=subset_ids, + fields=["_id"] + ) + version_ids = [ver["_id"] for ver in db_versions] + db_repres = get_representations( + self.project_name, + version_ids=version_ids, + fields=["_id"] + ) + repre_ids = [repre["_id"] for repre in db_repres] to_delete.extend(subset_ids) to_delete.extend(version_ids) to_delete.extend(repre_ids) - self.dbcon.delete_many({"_id": {"$in": to_delete}}) + if to_delete: + self.dbcon.delete_many({"_id": {"$in": to_delete}}) # Probably deprecated def _check_changeability(self, parent_id=None): @@ -2779,8 +2782,7 @@ def regex_duplicate_interface(self): def report(self): items = [] - project_name = self.entities_dict[self.ft_project_id]["name"] - title = "Synchronization report ({}):".format(project_name) + title = "Synchronization report ({}):".format(self.project_name) keys = ["error", "warning", "info"] for key in keys: diff --git a/openpype/modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py b/openpype/modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py index 73398941eb9..1a5d74bf260 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py @@ -3,7 +3,8 @@ import six import pyblish.api from copy import deepcopy -from openpype.pipeline import legacy_io +from openpype.client import get_asset_by_id + # Copy of constant `openpype_modules.ftrack.lib.avalon_sync.CUST_ATTR_AUTO_SYNC` CUST_ATTR_AUTO_SYNC = "avalon_auto_sync" @@ -82,9 +83,6 @@ def process(self, context): auto_sync_state = project[ "custom_attributes"][CUST_ATTR_AUTO_SYNC] - if not legacy_io.Session: - legacy_io.install() - self.ft_project = None # disable termporarily ftrack project's autosyncing @@ -93,14 +91,14 @@ def process(self, context): try: # import ftrack hierarchy - self.import_to_ftrack(hierarchy_context) + self.import_to_ftrack(project_name, hierarchy_context) except Exception: raise finally: if auto_sync_state: self.auto_sync_on(project) - def import_to_ftrack(self, input_data, parent=None): + def import_to_ftrack(self, project_name, input_data, parent=None): # Prequery hiearchical custom attributes hier_custom_attributes = get_pype_attr(self.session)[1] hier_attr_by_key = { @@ -222,7 +220,7 @@ def import_to_ftrack(self, input_data, parent=None): six.reraise(tp, value, tb) # Incoming links. - self.create_links(entity_data, entity) + self.create_links(project_name, entity_data, entity) try: self.session.commit() except Exception: @@ -255,9 +253,9 @@ def import_to_ftrack(self, input_data, parent=None): # Import children. if 'childs' in entity_data: self.import_to_ftrack( - entity_data['childs'], entity) + project_name, entity_data['childs'], entity) - def create_links(self, entity_data, entity): + def create_links(self, project_name, entity_data, entity): # Clear existing links. for link in entity.get("incoming_links", []): self.session.delete(link) @@ -270,9 +268,15 @@ def create_links(self, entity_data, entity): six.reraise(tp, value, tb) # Create new links. - for input in entity_data.get("inputs", []): - input_id = legacy_io.find_one({"_id": input})["data"]["ftrackId"] - assetbuild = self.session.get("AssetBuild", input_id) + for asset_id in entity_data.get("inputs", []): + asset_doc = get_asset_by_id(project_name, asset_id) + ftrack_id = None + if asset_doc: + ftrack_id = asset_doc["data"].get("ftrackId") + if not ftrack_id: + continue + + assetbuild = self.session.get("AssetBuild", ftrack_id) self.log.debug( "Creating link from {0} to {1}".format( assetbuild["name"], entity["name"]