diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py b/openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py index 3f63ce6face..868bbb84630 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py @@ -2,7 +2,10 @@ import json import collections import ftrack_api -from openpype_modules.ftrack.lib import ServerAction +from openpype_modules.ftrack.lib import ( + ServerAction, + query_custom_attributes +) class PushHierValuesToNonHier(ServerAction): @@ -51,10 +54,6 @@ class PushHierValuesToNonHier(ServerAction): " from CustomAttributeConfiguration" " where key in ({})" ) - cust_attr_value_query = ( - "select value, entity_id from CustomAttributeValue" - " where entity_id in ({}) and configuration_id in ({})" - ) # configurable settings_key = "sync_hier_entity_attributes" @@ -344,25 +343,11 @@ def get_hier_values( all_ids_with_parents.add(parent_id) _entity_id = parent_id - joined_entity_ids = self.join_query_keys(all_ids_with_parents) - - hier_attr_ids = self.join_query_keys( - tuple(hier_attr["id"] for hier_attr in hier_attrs) - ) + hier_attr_ids = tuple(hier_attr["id"] for hier_attr in hier_attrs) hier_attrs_key_by_id = { hier_attr["id"]: hier_attr["key"] for hier_attr in hier_attrs } - call_expr = [{ - "action": "query", - "expression": self.cust_attr_value_query.format( - joined_entity_ids, hier_attr_ids - ) - }] - if hasattr(session, "call"): - [values] = session.call(call_expr) - else: - [values] = session._call(call_expr) values_per_entity_id = {} for entity_id in all_ids_with_parents: @@ -370,7 +355,10 @@ def get_hier_values( for key in hier_attrs_key_by_id.values(): values_per_entity_id[entity_id][key] = None - for item in values["data"]: + values = query_custom_attributes( + session, all_ids_with_parents, hier_attr_ids, True + ) + for item in values: entity_id = item["entity_id"] key = hier_attrs_key_by_id[item["configuration_id"]] diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py b/openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py index 10b165e7f6c..0914933de4d 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py @@ -17,11 +17,6 @@ class PushFrameValuesToTaskEvent(BaseEvent): " (object_type_id in ({}) or is_hierarchical is true)" ) - cust_attr_query = ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" - ) - _cached_task_object_id = None _cached_interest_object_ids = None _cached_user_id = None @@ -273,16 +268,23 @@ def process_task_parent_change( hier_attr_ids.append(attr_id) conf_ids = list(hier_attr_ids) + task_conf_ids = [] for key, attr_id in task_attrs.items(): attr_key_by_id[attr_id] = key nonhier_id_by_key[key] = attr_id conf_ids.append(attr_id) + task_conf_ids.append(attr_id) # Query custom attribute values # - result does not contain values for all entities only result of # query callback to ftrack server result = query_custom_attributes( - session, conf_ids, whole_hierarchy_ids + session, list(hier_attr_ids), whole_hierarchy_ids, True + ) + result.extend( + query_custom_attributes( + session, task_conf_ids, whole_hierarchy_ids, False + ) ) # Prepare variables where result will be stored @@ -547,7 +549,7 @@ def finalize_attribute_changes( ) attr_ids = set(attr_id_to_key.keys()) - current_values_by_id = self.current_values( + current_values_by_id = self.get_current_values( session, attr_ids, entity_ids, task_entity_ids, hier_attrs ) @@ -642,27 +644,17 @@ def filter_changes( return interesting_data, changed_keys_by_object_id - def current_values( + def get_current_values( self, session, attr_ids, entity_ids, task_entity_ids, hier_attrs ): current_values_by_id = {} if not attr_ids or not entity_ids: return current_values_by_id - joined_conf_ids = self.join_query_keys(attr_ids) - joined_entity_ids = self.join_query_keys(entity_ids) - call_expr = [{ - "action": "query", - "expression": self.cust_attr_query.format( - joined_entity_ids, joined_conf_ids - ) - }] - if hasattr(session, "call"): - [values] = session.call(call_expr) - else: - [values] = session._call(call_expr) - - for item in values["data"]: + values = query_custom_attributes( + session, attr_ids, entity_ids, True + ) + for item in values: entity_id = item["entity_id"] attr_id = item["configuration_id"] if entity_id in task_entity_ids and attr_id in hier_attrs: diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py b/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py index 83132acd85b..9610e7f5de3 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py @@ -128,7 +128,7 @@ def _update_in_links(self, session, ftrack_ids, project_id): def _get_mongo_ids_by_ftrack_ids(self, session, attr_id, ftrack_ids): output = query_custom_attributes( - session, [attr_id], ftrack_ids + session, [attr_id], ftrack_ids, True ) mongo_id_by_ftrack_id = {} for item in output: diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 200e6620be6..9f85000dbba 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -17,6 +17,7 @@ from openpype_modules.ftrack.lib import ( get_openpype_attr, + query_custom_attributes, CUST_ATTR_ID_KEY, CUST_ATTR_AUTO_SYNC, @@ -2130,22 +2131,12 @@ def process_hier_cleanup(self): for key in hier_cust_attrs_keys: configuration_ids.add(hier_attr_id_by_key[key]) - entity_ids_joined = self.join_query_keys(cust_attrs_ftrack_ids) - attributes_joined = self.join_query_keys(configuration_ids) - - queries = [{ - "action": "query", - "expression": ( - "select value, entity_id, configuration_id" - " from CustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" - ).format(entity_ids_joined, attributes_joined) - }] - - if hasattr(self.process_session, "call"): - [values] = self.process_session.call(queries) - else: - [values] = self.process_session._call(queries) + values = query_custom_attributes( + self.process_session, + configuration_ids, + cust_attrs_ftrack_ids, + True + ) ftrack_project_id = self.cur_project["id"] @@ -2170,7 +2161,7 @@ def process_hier_cleanup(self): # PREPARE DATA BEFORE THIS avalon_hier = [] - for item in values["data"]: + for item in values: value = item["value"] if value is None: continue diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py index dc97ed972d8..f06162bfda7 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py @@ -19,8 +19,8 @@ class CleanHierarchicalAttrsAction(BaseAction): " from TypedContext where project_id is \"{}\"" ) cust_attr_query = ( - "select value, entity_id from CustomAttributeValue " - "where entity_id in ({}) and configuration_id is \"{}\"" + "select value, entity_id from CustomAttributeValue" + " where entity_id in ({}) and configuration_id is \"{}\"" ) settings_key = "clean_hierarchical_attr" @@ -65,17 +65,14 @@ def launch(self, session, entities, event): ) ) configuration_id = attr["id"] - call_expr = [{ - "action": "query", - "expression": self.cust_attr_query.format( + values = session.query( + self.cust_attr_query.format( entity_ids_joined, configuration_id ) - }] - - [values] = self.session.call(call_expr) + ).all() data = {} - for item in values["data"]: + for item in values: value = item["value"] if value is None: data[item["entity_id"]] = value @@ -90,10 +87,10 @@ def launch(self, session, entities, event): len(data), configuration_key )) for entity_id, value in data.items(): - entity_key = collections.OrderedDict({ - "configuration_id": configuration_id, - "entity_id": entity_id - }) + entity_key = collections.OrderedDict(( + ("configuration_id", configuration_id), + ("entity_id", entity_id) + )) session.recorded_operations.push( ftrack_api.operation.DeleteEntityOperation( "CustomAttributeValue", diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py index 0bd243ab4cd..cb5b88ad50c 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py @@ -306,8 +306,8 @@ def convert_mongo_id_to_hierarchical( } cust_attr_query = ( - "select value, entity_id from ContextCustomAttributeValue " - "where configuration_id is {}" + "select value, entity_id from CustomAttributeValue" + " where configuration_id is {}" ) for attr_def in object_type_attrs: attr_ent_type = attr_def["entity_type"] @@ -328,21 +328,14 @@ def convert_mongo_id_to_hierarchical( self.log.debug(( "Converting Avalon MongoID attr for Entity type \"{}\"." ).format(entity_type_label)) - - call_expr = [{ - "action": "query", - "expression": cust_attr_query.format(attr_def["id"]) - }] - if hasattr(session, "call"): - [values] = session.call(call_expr) - else: - [values] = session._call(call_expr) - - for value in values["data"]: - table_values = collections.OrderedDict({ - "configuration_id": hierarchical_attr["id"], - "entity_id": value["entity_id"] - }) + values = session.query( + cust_attr_query.format(attr_def["id"]) + ).all() + for value in values: + table_values = collections.OrderedDict([ + ("configuration_id", hierarchical_attr["id"]), + ("entity_id", value["entity_id"]) + ]) session.recorded_operations.push( ftrack_api.operation.UpdateEntityOperation( diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index 38ec02749a9..5c38df2e031 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -303,9 +303,10 @@ def on_project_anatomy_save( # TODO add add permissions check # TODO add value validations # - value type and list items - entity_key = collections.OrderedDict() - entity_key["configuration_id"] = configuration["id"] - entity_key["entity_id"] = project_id + entity_key = collections.OrderedDict([ + ("configuration_id", configuration["id"]), + ("entity_id", project_id) + ]) session.recorded_operations.push( ftrack_api.operation.UpdateEntityOperation( diff --git a/openpype/modules/default_modules/ftrack/lib/avalon_sync.py b/openpype/modules/default_modules/ftrack/lib/avalon_sync.py index 66cf7645c24..06e87842872 100644 --- a/openpype/modules/default_modules/ftrack/lib/avalon_sync.py +++ b/openpype/modules/default_modules/ftrack/lib/avalon_sync.py @@ -1,11 +1,8 @@ -import os import re import json import collections import copy -import six - from avalon.api import AvalonMongoDB import avalon @@ -18,7 +15,7 @@ from openpype.lib import ApplicationManager from .constants import CUST_ATTR_ID_KEY -from .custom_attributes import get_openpype_attr +from .custom_attributes import get_openpype_attr, query_custom_attributes from bson.objectid import ObjectId from bson.errors import InvalidId @@ -235,33 +232,19 @@ def get_hierarchical_attributes_values( entity_ids = [item["id"] for item in entity["link"]] - join_ent_ids = join_query_keys(entity_ids) - join_attribute_ids = join_query_keys(attr_key_by_id.keys()) - - queries = [] - queries.append({ - "action": "query", - "expression": ( - "select value, configuration_id, entity_id" - " from CustomAttributeValue" - " where entity_id in ({}) and configuration_id in ({})" - ).format(join_ent_ids, join_attribute_ids) - }) - - if hasattr(session, "call"): - [values] = session.call(queries) - else: - [values] = session._call(queries) + values = query_custom_attributes( + session, list(attr_key_by_id.keys()), entity_ids, True + ) hier_values = {} for key, val in defaults.items(): hier_values[key] = val - if not values["data"]: + if not values: return hier_values values_by_entity_id = collections.defaultdict(dict) - for item in values["data"]: + for item in values: value = item["value"] if value is None: continue @@ -861,33 +844,6 @@ def filter_by_selection(self, event): self.entities_dict[parent_id]["children"].remove(ftrack_id) - def _query_custom_attributes(self, session, conf_ids, entity_ids): - output = [] - # Prepare values to query - attributes_joined = join_query_keys(conf_ids) - attributes_len = len(conf_ids) - chunk_size = int(5000 / attributes_len) - for idx in range(0, len(entity_ids), chunk_size): - entity_ids_joined = join_query_keys( - entity_ids[idx:idx + chunk_size] - ) - - call_expr = [{ - "action": "query", - "expression": ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" - ).format(entity_ids_joined, attributes_joined) - }] - if hasattr(session, "call"): - [result] = session.call(call_expr) - else: - [result] = session._call(call_expr) - - for item in result["data"]: - output.append(item) - return output - def set_cutom_attributes(self): self.log.debug("* Preparing custom attributes") # Get custom attributes and values @@ -994,7 +950,7 @@ def set_cutom_attributes(self): copy.deepcopy(prepared_avalon_attr_ca_id) ) - items = self._query_custom_attributes( + items = query_custom_attributes( self.session, list(attribute_key_by_id.keys()), sync_ids @@ -1082,10 +1038,11 @@ def set_hierarchical_attribute( for key, val in prepare_dict_avalon.items(): entity_dict["avalon_attrs"][key] = val - items = self._query_custom_attributes( + items = query_custom_attributes( self.session, list(attribute_key_by_id.keys()), - sync_ids + sync_ids, + True ) avalon_hier = [] @@ -1806,10 +1763,10 @@ def prepare_changes(self): configuration_id = self.entities_dict[ftrack_id][ "avalon_attrs_id"][CUST_ATTR_ID_KEY] - _entity_key = collections.OrderedDict({ - "configuration_id": configuration_id, - "entity_id": ftrack_id - }) + _entity_key = collections.OrderedDict([ + ("configuration_id", configuration_id), + ("entity_id", ftrack_id) + ]) self.session.recorded_operations.push( ftrack_api.operation.UpdateEntityOperation( diff --git a/openpype/modules/default_modules/ftrack/lib/custom_attributes.py b/openpype/modules/default_modules/ftrack/lib/custom_attributes.py index 53facd4ab22..8b1cbb8e54a 100644 --- a/openpype/modules/default_modules/ftrack/lib/custom_attributes.py +++ b/openpype/modules/default_modules/ftrack/lib/custom_attributes.py @@ -88,26 +88,36 @@ def join_query_keys(keys): return ",".join(["\"{}\"".format(key) for key in keys]) -def query_custom_attributes(session, conf_ids, entity_ids, table_name=None): +def query_custom_attributes( + session, conf_ids, entity_ids, only_set_values=False +): """Query custom attribute values from ftrack database. Using ftrack call method result may differ based on used table name and version of ftrack server. + For hierarchical attributes you shou always use `only_set_values=True` + otherwise result will be default value of custom attribute and it would not + be possible to differentiate if value is set on entity or default value is + used. + Args: session(ftrack_api.Session): Connected ftrack session. conf_id(list, set, tuple): Configuration(attribute) ids which are queried. entity_ids(list, set, tuple): Entity ids for which are values queried. - table_name(str): Table nam from which values are queried. Not - recommended to change until you know what it means. + only_set_values(bool): Entities that don't have explicitly set + value won't return a value. If is set to False then default custom + attribute value is returned if value is not set. """ output = [] # Just skip if not conf_ids or not entity_ids: return output - if table_name is None: + if only_set_values: + table_name = "CustomAttributeValue" + else: table_name = "ContextCustomAttributeValue" # Prepare values to query @@ -122,19 +132,16 @@ def query_custom_attributes(session, conf_ids, entity_ids, table_name=None): entity_ids_joined = join_query_keys( entity_ids[idx:idx + chunk_size] ) - - call_expr = [{ - "action": "query", - "expression": ( - "select value, entity_id from {}" - " where entity_id in ({}) and configuration_id in ({})" - ).format(table_name, entity_ids_joined, attributes_joined) - }] - if hasattr(session, "call"): - [result] = session.call(call_expr) - else: - [result] = session._call(call_expr) - - for item in result["data"]: - output.append(item) + output.extend( + session.query( + ( + "select value, entity_id from {}" + " where entity_id in ({}) and configuration_id in ({})" + ).format( + table_name, + entity_ids_joined, + attributes_joined + ) + ).all() + ) return output