From bc8d66fdc1a25be8cb2e1afcf6d122b263b90125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 28 Oct 2020 13:03:11 +0100 Subject: [PATCH 01/47] extracted few things from lib.py --- pype/lib.py | 109 ++------------------------------------- pype/lib/__init__.py | 11 ++++ pype/lib/hooks.py | 71 +++++++++++++++++++++++++ pype/lib/plugin_tools.py | 59 +++++++++++++++++++++ 4 files changed, 144 insertions(+), 106 deletions(-) create mode 100644 pype/lib/__init__.py create mode 100644 pype/lib/hooks.py create mode 100644 pype/lib/plugin_tools.py diff --git a/pype/lib.py b/pype/lib.py index afcfa98307a..ef3c1c1ae5b 100644 --- a/pype/lib.py +++ b/pype/lib.py @@ -1,6 +1,6 @@ import os import sys -import types + import re import uuid import json @@ -11,15 +11,14 @@ import contextlib import subprocess import getpass -import inspect import acre import platform -from abc import ABCMeta, abstractmethod from avalon import io, pipeline -import six + import avalon.api from .api import config, Anatomy, Logger +from .lib import execute_hook log = logging.getLogger(__name__) @@ -551,53 +550,6 @@ def set_io_database(): io.install() -def filter_pyblish_plugins(plugins): - """ - This servers as plugin filter / modifier for pyblish. It will load plugin - definitions from presets and filter those needed to be excluded. - - :param plugins: Dictionary of plugins produced by :mod:`pyblish-base` - `discover()` method. - :type plugins: Dict - """ - from pyblish import api - - host = api.current_host() - - presets = config.get_presets().get('plugins', {}) - - # iterate over plugins - for plugin in plugins[:]: - # skip if there are no presets to process - if not presets: - continue - - file = os.path.normpath(inspect.getsourcefile(plugin)) - file = os.path.normpath(file) - - # host determined from path - host_from_file = file.split(os.path.sep)[-3:-2][0] - plugin_kind = file.split(os.path.sep)[-2:-1][0] - - try: - config_data = presets[host]["publish"][plugin.__name__] - except KeyError: - try: - config_data = presets[host_from_file][plugin_kind][plugin.__name__] # noqa: E501 - except KeyError: - continue - - for option, value in config_data.items(): - if option == "enabled" and value is False: - log.info('removing plugin {}'.format(plugin.__name__)) - plugins.remove(plugin) - else: - log.info('setting {}:{} on plugin {}'.format( - option, value, plugin.__name__)) - - setattr(plugin, option, value) - - def get_subsets(asset_name, regex_filter=None, version=None, @@ -715,61 +667,6 @@ def __repr__(self): return "".format(str(self.identifier)) -def execute_hook(hook, *args, **kwargs): - """ - This will load hook file, instantiate class and call `execute` method - on it. Hook must be in a form: - - `$PYPE_SETUP_PATH/repos/pype/path/to/hook.py/HookClass` - - This will load `hook.py`, instantiate HookClass and then execute_hook - `execute(*args, **kwargs)` - - :param hook: path to hook class - :type hook: str - """ - - class_name = hook.split("/")[-1] - - abspath = os.path.join(os.getenv('PYPE_SETUP_PATH'), - 'repos', 'pype', *hook.split("/")[:-1]) - - mod_name, mod_ext = os.path.splitext(os.path.basename(abspath)) - - if not mod_ext == ".py": - return False - - module = types.ModuleType(mod_name) - module.__file__ = abspath - - try: - with open(abspath) as f: - six.exec_(f.read(), module.__dict__) - - sys.modules[abspath] = module - - except Exception as exp: - log.exception("loading hook failed: {}".format(exp), - exc_info=True) - return False - - obj = getattr(module, class_name) - hook_obj = obj() - ret_val = hook_obj.execute(*args, **kwargs) - return ret_val - - -@six.add_metaclass(ABCMeta) -class PypeHook: - - def __init__(self): - pass - - @abstractmethod - def execute(self, *args, **kwargs): - pass - - def get_linked_assets(asset_entity): """Return linked assets for `asset_entity`.""" inputs = asset_entity["data"].get("inputs", []) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py new file mode 100644 index 00000000000..51f305950c3 --- /dev/null +++ b/pype/lib/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +"""Pype lib module.""" +from .hooks import PypeHook, execute_hook +from .plugin_tools import filter_pyblish_plugins + +__all__ = [ + "PypeHook", + "execute_hook", + + "filter_pyblish_plugins" +] diff --git a/pype/lib/hooks.py b/pype/lib/hooks.py new file mode 100644 index 00000000000..425ad36342d --- /dev/null +++ b/pype/lib/hooks.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Package containing code for handling hooks.""" +import os +import sys +import types +import logging +from abc import ABCMeta, abstractmethod + +import six + + +log = logging.getLogger(__name__) + + +@six.add_metaclass(ABCMeta) +class PypeHook: + """Abstract class from all hooks should inherit.""" + + def __init__(self): + """Constructor.""" + pass + + @abstractmethod + def execute(self, *args, **kwargs): + """Abstract execute method.""" + pass + + +def execute_hook(hook, *args, **kwargs): + """Execute hook with arguments. + + This will load hook file, instantiate class and call + :meth:`PypeHook.execute` method on it. Hook must be in a form:: + + $PYPE_SETUP_PATH/repos/pype/path/to/hook.py/HookClass + + This will load `hook.py`, instantiate HookClass and then execute_hook + `execute(*args, **kwargs)` + + Args: + hook (str): path to hook class. + + """ + class_name = hook.split("/")[-1] + + abspath = os.path.join(os.getenv('PYPE_SETUP_PATH'), + 'repos', 'pype', *hook.split("/")[:-1]) + + mod_name, mod_ext = os.path.splitext(os.path.basename(abspath)) + + if not mod_ext == ".py": + return False + + module = types.ModuleType(mod_name) + module.__file__ = abspath + + try: + with open(abspath) as f: + six.exec_(f.read(), module.__dict__) + + sys.modules[abspath] = module + + except Exception as exp: + log.exception("loading hook failed: {}".format(exp), + exc_info=True) + return False + + obj = getattr(module, class_name) + hook_obj = obj() + ret_val = hook_obj.execute(*args, **kwargs) + return ret_val diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py new file mode 100644 index 00000000000..498da075c55 --- /dev/null +++ b/pype/lib/plugin_tools.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +"""Avalon/Pyblish plugin tools.""" +import os +import inspect +import logging + +from .api import config + + +log = logging.getLogger(__name__) + + +def filter_pyblish_plugins(plugins): + """Filter pyblish plugins by presets. + + This servers as plugin filter / modifier for pyblish. It will load plugin + definitions from presets and filter those needed to be excluded. + + Args: + plugins (dict): Dictionary of plugins produced by :mod:`pyblish-base` + `discover()` method. + + """ + from pyblish import api + + host = api.current_host() + + presets = config.get_presets().get('plugins', {}) + + # iterate over plugins + for plugin in plugins[:]: + # skip if there are no presets to process + if not presets: + continue + + file = os.path.normpath(inspect.getsourcefile(plugin)) + file = os.path.normpath(file) + + # host determined from path + host_from_file = file.split(os.path.sep)[-3:-2][0] + plugin_kind = file.split(os.path.sep)[-2:-1][0] + + try: + config_data = presets[host]["publish"][plugin.__name__] + except KeyError: + try: + config_data = presets[host_from_file][plugin_kind][plugin.__name__] # noqa: E501 + except KeyError: + continue + + for option, value in config_data.items(): + if option == "enabled" and value is False: + log.info('removing plugin {}'.format(plugin.__name__)) + plugins.remove(plugin) + else: + log.info('setting {}:{} on plugin {}'.format( + option, value, plugin.__name__)) + + setattr(plugin, option, value) From a2a10048b66e196922d5b3e823abc842edcccbb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 28 Oct 2020 15:11:39 +0100 Subject: [PATCH 02/47] create lib module --- pype/lib/__init__.py | 1 + pype/{lib.py => lib/lib_old.py} | 0 2 files changed, 1 insertion(+) create mode 100644 pype/lib/__init__.py rename pype/{lib.py => lib/lib_old.py} (100%) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py new file mode 100644 index 00000000000..ad42cd1deec --- /dev/null +++ b/pype/lib/__init__.py @@ -0,0 +1 @@ +from .lib_old import * diff --git a/pype/lib.py b/pype/lib/lib_old.py similarity index 100% rename from pype/lib.py rename to pype/lib/lib_old.py From eb9a67acf7a742000bec04a7b1303ca1e36fea25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 28 Oct 2020 15:16:13 +0100 Subject: [PATCH 03/47] few fixes --- pype/lib/lib_old.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index ef3c1c1ae5b..eb22556e2c1 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -1,6 +1,5 @@ import os import sys - import re import uuid import json @@ -13,12 +12,11 @@ import getpass import acre import platform +from . import execute_hook from avalon import io, pipeline - import avalon.api from .api import config, Anatomy, Logger -from .lib import execute_hook log = logging.getLogger(__name__) From c13e48c0262cb965f85e1a606fb2b943b9a3a0d7 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 28 Oct 2020 16:02:07 +0100 Subject: [PATCH 04/47] lib changed to lib/old_lib to have a starting point --- pype/lib/__init__.py | 39 ++++++++++++++++++++++++++++++++++++++- pype/lib/lib_old.py | 2 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index ad42cd1deec..d8ce8badbc0 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -1 +1,38 @@ -from .lib_old import * +# -*- coding: utf-8 -*- +"""Pype lib module.""" +from .lib_old import ( + _subprocess, + get_paths_from_environ, + get_ffmpeg_tool_path, + get_hierarchy, + add_tool_to_environment, + modified_environ, + pairwise, + grouper, + is_latest, + any_outdated, + _rreplace, + version_up, + switch_item, + _get_host_name, + get_asset, + get_project, + get_version_from_path, + get_last_version_from_path, + get_avalon_database, + set_io_database, + filter_pyblish_plugins, + get_subsets, + CustomNone, + execute_hook, + PypeHook, + get_linked_assets, + map_subsets_by_family, + BuildWorkfile, + ffprobe_streams, + source_hash, + get_latest_version, + ApplicationLaunchFailed, + launch_application, + ApplicationAction + ) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index afcfa98307a..e6de3882e95 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -19,7 +19,7 @@ from avalon import io, pipeline import six import avalon.api -from .api import config, Anatomy, Logger +from ..api import config, Anatomy, Logger log = logging.getLogger(__name__) From d299c21a0abc73a961b5095cb5c9f7dcd43bc072 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 28 Oct 2020 16:03:52 +0100 Subject: [PATCH 05/47] remove functions from old-lib import --- pype/lib/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 6286d26078d..12633959aac 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -21,11 +21,8 @@ get_last_version_from_path, get_avalon_database, set_io_database, - filter_pyblish_plugins, get_subsets, CustomNone, - execute_hook, - PypeHook, get_linked_assets, map_subsets_by_family, BuildWorkfile, From d11ef77236bee628c5f0547c9f7e3ba04c34f0c3 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 28 Oct 2020 16:07:09 +0100 Subject: [PATCH 06/47] fix relative imports for 2.x compatibility --- pype/lib/lib_old.py | 2 +- pype/lib/plugin_tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 224777b8ca3..08146309cc0 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -12,7 +12,7 @@ import getpass import acre import platform -from . import execute_hook +from pype.lib.hooks import execute_hook from avalon import io, pipeline import avalon.api diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py index 498da075c55..066f1ff20a5 100644 --- a/pype/lib/plugin_tools.py +++ b/pype/lib/plugin_tools.py @@ -4,7 +4,7 @@ import inspect import logging -from .api import config +from ..api import config log = logging.getLogger(__name__) From 98012be8cc0cecd75473ae4798959f93e058f4c2 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 3 Nov 2020 09:51:30 +0100 Subject: [PATCH 07/47] add deprecated.py --- pype/lib/deprecated.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pype/lib/deprecated.py diff --git a/pype/lib/deprecated.py b/pype/lib/deprecated.py new file mode 100644 index 00000000000..e69de29bb2d From 15c03de4852ee337cc4f80a46543929bb9a559d2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 10:54:10 +0100 Subject: [PATCH 08/47] moved `get_avalon_database` and `set_io_database` to deprecated --- pype/lib/__init__.py | 10 ++++++++-- pype/lib/deprecated.py | 26 ++++++++++++++++++++++++++ pype/lib/lib_old.py | 10 +--------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 12633959aac..4ca5fa999cd 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- """Pype lib module.""" + +from .deprecated import ( + get_avalon_database, + set_io_database +) from .lib_old import ( _subprocess, get_paths_from_environ, @@ -19,8 +24,6 @@ get_project, get_version_from_path, get_last_version_from_path, - get_avalon_database, - set_io_database, get_subsets, CustomNone, get_linked_assets, @@ -38,6 +41,9 @@ from .plugin_tools import filter_pyblish_plugins __all__ = [ + "get_avalon_database", + "set_io_database", + "PypeHook", "execute_hook", diff --git a/pype/lib/deprecated.py b/pype/lib/deprecated.py index e69de29bb2d..e7296f67ef4 100644 --- a/pype/lib/deprecated.py +++ b/pype/lib/deprecated.py @@ -0,0 +1,26 @@ +import os + +from avalon import io + + +def get_avalon_database(): + """Mongo database used in avalon's io. + + * Function is not used in pype 3.0 where was replaced with usage of + AvalonMongoDB. + """ + if io._database is None: + set_io_database() + return io._database + + +def set_io_database(): + """Set avalon's io context with environemnts. + + * Function is not used in pype 3.0 where was replaced with usage of + AvalonMongoDB. + """ + required_keys = ["AVALON_PROJECT", "AVALON_ASSET", "AVALON_SILO"] + for key in required_keys: + os.environ[key] = os.environ.get(key, "") + io.install() diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 08146309cc0..7284a1026d0 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -13,6 +13,7 @@ import acre import platform from pype.lib.hooks import execute_hook +from .deprecated import get_avalon_database from avalon import io, pipeline import avalon.api @@ -535,17 +536,8 @@ def get_last_version_from_path(path_dir, filter): return None -def get_avalon_database(): - if io._database is None: - set_io_database() - return io._database -def set_io_database(): - required_keys = ["AVALON_PROJECT", "AVALON_ASSET", "AVALON_SILO"] - for key in required_keys: - os.environ[key] = os.environ.get(key, "") - io.install() def get_subsets(asset_name, From 0096705347240e33fbf7473aacd8bc2745f9a51d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 10:54:28 +0100 Subject: [PATCH 09/47] removed unused `CustomNone` --- pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 39 --------------------------------------- 2 files changed, 40 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 4ca5fa999cd..bf3771c0e70 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -25,7 +25,6 @@ get_version_from_path, get_last_version_from_path, get_subsets, - CustomNone, get_linked_assets, map_subsets_by_family, BuildWorkfile, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 7284a1026d0..6b142c8325f 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -618,45 +618,6 @@ def get_subsets(asset_name, return output_dict -class CustomNone: - """Created object can be used as custom None (not equal to None). - - WARNING: Multiple created objects are not equal either. - Exmple: - >>> a = CustomNone() - >>> a == None - False - >>> b = CustomNone() - >>> a == b - False - >>> a == a - True - """ - - def __init__(self): - """Create uuid as identifier for custom None.""" - self.identifier = str(uuid.uuid4()) - - def __bool__(self): - """Return False (like default None).""" - return False - - def __eq__(self, other): - """Equality is compared by identifier value.""" - if type(other) == type(self): - if other.identifier == self.identifier: - return True - return False - - def __str__(self): - """Return value of identifier when converted to string.""" - return self.identifier - - def __repr__(self): - """Representation of custom None.""" - return "".format(str(self.identifier)) - - def get_linked_assets(asset_entity): """Return linked assets for `asset_entity`.""" inputs = asset_entity["data"].get("inputs", []) From dbfbe9983424db2af695badd2ec48201b435b1df Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 10:56:53 +0100 Subject: [PATCH 10/47] removed single line function `get_project` --- pype/hosts/hiero/lib.py | 4 +++- pype/hosts/maya/lib.py | 4 ++-- pype/hosts/nuke/lib.py | 2 +- pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 5 ----- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pype/hosts/hiero/lib.py b/pype/hosts/hiero/lib.py index db7199a190e..a508343bfaf 100644 --- a/pype/hosts/hiero/lib.py +++ b/pype/hosts/hiero/lib.py @@ -4,6 +4,7 @@ import hiero import pyblish.api import avalon.api as avalon +import avalon.io from avalon.vendor.Qt import (QtWidgets, QtGui) import pype.api as pype from pype.api import Logger, Anatomy @@ -58,7 +59,8 @@ def sync_avalon_data_to_workfile(): project.setProjectRoot(active_project_root) # get project data from avalon db - project_data = pype.get_project()["data"] + project_doc = avalon.io.find_one({"type": "project"}) + project_data = project_doc["data"] log.debug("project_data: {}".format(project_data)) diff --git a/pype/hosts/maya/lib.py b/pype/hosts/maya/lib.py index 2dda198d452..e7ca5ec4dc5 100644 --- a/pype/hosts/maya/lib.py +++ b/pype/hosts/maya/lib.py @@ -1857,8 +1857,8 @@ def set_context_settings(): """ # Todo (Wijnand): apply renderer and resolution of project - - project_data = lib.get_project()["data"] + project_doc = io.find_one({"type": "project"}) + project_data = project_doc["data"] asset_data = lib.get_asset()["data"] # Set project fps diff --git a/pype/hosts/nuke/lib.py b/pype/hosts/nuke/lib.py index 8fd84b85559..24cd4f9a97f 100644 --- a/pype/hosts/nuke/lib.py +++ b/pype/hosts/nuke/lib.py @@ -195,7 +195,7 @@ def format_anatomy(data): if not version: file = script_name() data["version"] = pype.get_version_from_path(file) - project_document = pype.get_project() + project_document = io.find_one({"type": "project"}) data.update({ "subset": data["avalon"]["subset"], "asset": data["avalon"]["asset"], diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index bf3771c0e70..b2bbd9e60de 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -21,7 +21,6 @@ switch_item, _get_host_name, get_asset, - get_project, get_version_from_path, get_last_version_from_path, get_subsets, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 6b142c8325f..747865f08d7 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -472,11 +472,6 @@ def get_asset(asset_name=None): return asset_document -def get_project(): - io.install() - return io.find_one({"type": "project"}) - - def get_version_from_path(file): """ Finds version number in file path string From 83578a90b27f1f327b99c46e1306f7687d55c376 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:01:25 +0100 Subject: [PATCH 11/47] removed unused import --- pype/lib/lib_old.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 747865f08d7..309ea4190f0 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -1,7 +1,6 @@ import os import sys import re -import uuid import json import collections import logging From d46f6fba12a547daa5c7e4c49f915a5f05228979 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:11:02 +0100 Subject: [PATCH 12/47] moved independent imports as first in lib's __init__ --- pype/lib/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index b2bbd9e60de..a9cbc8c6caf 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -5,6 +5,9 @@ get_avalon_database, set_io_database ) + +from .hooks import PypeHook, execute_hook +from .plugin_tools import filter_pyblish_plugins from .lib_old import ( _subprocess, get_paths_from_environ, @@ -35,8 +38,6 @@ ApplicationAction ) -from .hooks import PypeHook, execute_hook -from .plugin_tools import filter_pyblish_plugins __all__ = [ "get_avalon_database", From 2ce0e7a46549659c198340dc8139fbaaa0960802 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:13:05 +0100 Subject: [PATCH 13/47] removed `get_project` import from pype.api --- pype/api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/api.py b/pype/api.py index c1bf84b4efd..2c7dfa73f07 100644 --- a/pype/api.py +++ b/pype/api.py @@ -39,7 +39,6 @@ from .lib import ( version_up, get_asset, - get_project, get_hierarchy, get_subsets, get_version_from_path, @@ -88,7 +87,6 @@ # get contextual data "version_up", - "get_project", "get_hierarchy", "get_asset", "get_subsets", From 9887dc40644207ac3f94af552535a7a81394bc9e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:13:35 +0100 Subject: [PATCH 14/47] moved applications import from lib_old --- pype/lib/__init__.py | 20 +- pype/lib/applications.py | 385 +++++++++++++++++++++++++++++++++++++++ pype/lib/lib_old.py | 376 -------------------------------------- 3 files changed, 399 insertions(+), 382 deletions(-) create mode 100644 pype/lib/applications.py diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index a9cbc8c6caf..6620bd5a122 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -7,7 +7,15 @@ ) from .hooks import PypeHook, execute_hook + +from .applications import ( + ApplicationLaunchFailed, + launch_application, + ApplicationAction +) + from .plugin_tools import filter_pyblish_plugins + from .lib_old import ( _subprocess, get_paths_from_environ, @@ -32,12 +40,8 @@ BuildWorkfile, ffprobe_streams, source_hash, - get_latest_version, - ApplicationLaunchFailed, - launch_application, - ApplicationAction - ) - + get_latest_version +) __all__ = [ "get_avalon_database", @@ -46,5 +50,9 @@ "PypeHook", "execute_hook", + "ApplicationLaunchFailed", + "launch_application", + "ApplicationAction", + "filter_pyblish_plugins" ] diff --git a/pype/lib/applications.py b/pype/lib/applications.py new file mode 100644 index 00000000000..f4861ae8668 --- /dev/null +++ b/pype/lib/applications.py @@ -0,0 +1,385 @@ +import os +import sys +import getpass +import copy +import platform +import logging + +import acre + +import avalon.lib + +from ..api import Anatomy, Logger, config +from .hooks import execute_hook +from .deprecated import get_avalon_database + +log = logging.getLogger(__name__) + + +class ApplicationLaunchFailed(Exception): + pass + + +def launch_application(project_name, asset_name, task_name, app_name): + database = get_avalon_database() + project_document = database[project_name].find_one({"type": "project"}) + asset_document = database[project_name].find_one({ + "type": "asset", + "name": asset_name + }) + + asset_doc_parents = asset_document["data"].get("parents") + hierarchy = "/".join(asset_doc_parents) + + app_def = avalon.lib.get_application(app_name) + app_label = app_def.get("ftrack_label", app_def.get("label", app_name)) + + host_name = app_def["application_dir"] + data = { + "project": { + "name": project_document["name"], + "code": project_document["data"].get("code") + }, + "task": task_name, + "asset": asset_name, + "app": host_name, + "hierarchy": hierarchy + } + + try: + anatomy = Anatomy(project_name) + anatomy_filled = anatomy.format(data) + workdir = os.path.normpath(anatomy_filled["work"]["folder"]) + + except Exception as exc: + raise ApplicationLaunchFailed( + "Error in anatomy.format: {}".format(str(exc)) + ) + + try: + os.makedirs(workdir) + except FileExistsError: + pass + + last_workfile_path = None + extensions = avalon.api.HOST_WORKFILE_EXTENSIONS.get(host_name) + if extensions: + # Find last workfile + file_template = anatomy.templates["work"]["file"] + data.update({ + "version": 1, + "user": os.environ.get("PYPE_USERNAME") or getpass.getuser(), + "ext": extensions[0] + }) + + last_workfile_path = avalon.api.last_workfile( + workdir, file_template, data, extensions, True + ) + + # set environments for Avalon + prep_env = copy.deepcopy(os.environ) + prep_env.update({ + "AVALON_PROJECT": project_name, + "AVALON_ASSET": asset_name, + "AVALON_TASK": task_name, + "AVALON_APP": host_name, + "AVALON_APP_NAME": app_name, + "AVALON_HIERARCHY": hierarchy, + "AVALON_WORKDIR": workdir + }) + + start_last_workfile = avalon.api.should_start_last_workfile( + project_name, host_name, task_name + ) + # Store boolean as "0"(False) or "1"(True) + prep_env["AVALON_OPEN_LAST_WORKFILE"] = ( + str(int(bool(start_last_workfile))) + ) + + if ( + start_last_workfile + and last_workfile_path + and os.path.exists(last_workfile_path) + ): + prep_env["AVALON_LAST_WORKFILE"] = last_workfile_path + + prep_env.update(anatomy.roots_obj.root_environments()) + + # collect all the 'environment' attributes from parents + tools_attr = [prep_env["AVALON_APP"], prep_env["AVALON_APP_NAME"]] + tools_env = asset_document["data"].get("tools_env") or [] + tools_attr.extend(tools_env) + + tools_env = acre.get_tools(tools_attr) + env = acre.compute(tools_env) + env = acre.merge(env, current_env=dict(prep_env)) + + # Get path to execute + st_temp_path = os.environ["PYPE_CONFIG"] + os_plat = platform.system().lower() + + # Path to folder with launchers + path = os.path.join(st_temp_path, "launchers", os_plat) + + # Full path to executable launcher + execfile = None + + launch_hook = app_def.get("launch_hook") + if launch_hook: + log.info("launching hook: {}".format(launch_hook)) + ret_val = execute_hook(launch_hook, env=env) + if not ret_val: + raise ApplicationLaunchFailed( + "Hook didn't finish successfully {}".format(app_label) + ) + + if sys.platform == "win32": + for ext in os.environ["PATHEXT"].split(os.pathsep): + fpath = os.path.join(path.strip('"'), app_def["executable"] + ext) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + execfile = fpath + break + + # Run SW if was found executable + if execfile is None: + raise ApplicationLaunchFailed( + "We didn't find launcher for {}".format(app_label) + ) + + popen = avalon.lib.launch( + executable=execfile, args=[], environment=env + ) + + elif ( + sys.platform.startswith("linux") + or sys.platform.startswith("darwin") + ): + execfile = os.path.join(path.strip('"'), app_def["executable"]) + # Run SW if was found executable + if execfile is None: + raise ApplicationLaunchFailed( + "We didn't find launcher for {}".format(app_label) + ) + + if not os.path.isfile(execfile): + raise ApplicationLaunchFailed( + "Launcher doesn't exist - {}".format(execfile) + ) + + try: + fp = open(execfile) + except PermissionError as perm_exc: + raise ApplicationLaunchFailed( + "Access denied on launcher {} - {}".format(execfile, perm_exc) + ) + + fp.close() + # check executable permission + if not os.access(execfile, os.X_OK): + raise ApplicationLaunchFailed( + "No executable permission - {}".format(execfile) + ) + + popen = avalon.lib.launch( # noqa: F841 + "/usr/bin/env", args=["bash", execfile], environment=env + ) + return popen + + +class ApplicationAction(avalon.api.Action): + """Default application launcher + + This is a convenience application Action that when "config" refers to a + parsed application `.toml` this can launch the application. + + """ + _log = None + config = None + group = None + variant = None + required_session_keys = ( + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK" + ) + + @property + def log(self): + if self._log is None: + self._log = Logger().get_logger(self.__class__.__name__) + return self._log + + def is_compatible(self, session): + for key in self.required_session_keys: + if key not in session: + return False + return True + + def process(self, session, **kwargs): + """Process the full Application action""" + + project_name = session["AVALON_PROJECT"] + asset_name = session["AVALON_ASSET"] + task_name = session["AVALON_TASK"] + launch_application( + project_name, asset_name, task_name, self.name + ) + + self._ftrack_after_launch_procedure( + project_name, asset_name, task_name + ) + + def _ftrack_after_launch_procedure( + self, project_name, asset_name, task_name + ): + # TODO move to launch hook + required_keys = ("FTRACK_SERVER", "FTRACK_API_USER", "FTRACK_API_KEY") + for key in required_keys: + if not os.environ.get(key): + self.log.debug(( + "Missing required environment \"{}\"" + " for Ftrack after launch procedure." + ).format(key)) + return + + try: + import ftrack_api + session = ftrack_api.Session(auto_connect_event_hub=True) + self.log.debug("Ftrack session created") + except Exception: + self.log.warning("Couldn't create Ftrack session") + return + + try: + entity = self._find_ftrack_task_entity( + session, project_name, asset_name, task_name + ) + self._ftrack_status_change(session, entity, project_name) + self._start_timer(session, entity, ftrack_api) + except Exception: + self.log.warning( + "Couldn't finish Ftrack procedure.", exc_info=True + ) + return + + finally: + session.close() + + def _find_ftrack_task_entity( + self, session, project_name, asset_name, task_name + ): + project_entity = session.query( + "Project where full_name is \"{}\"".format(project_name) + ).first() + if not project_entity: + self.log.warning( + "Couldn't find project \"{}\" in Ftrack.".format(project_name) + ) + return + + potential_task_entities = session.query(( + "TypedContext where parent.name is \"{}\" and project_id is \"{}\"" + ).format(asset_name, project_entity["id"])).all() + filtered_entities = [] + for _entity in potential_task_entities: + if ( + _entity.entity_type.lower() == "task" + and _entity["name"] == task_name + ): + filtered_entities.append(_entity) + + if not filtered_entities: + self.log.warning(( + "Couldn't find task \"{}\" under parent \"{}\" in Ftrack." + ).format(task_name, asset_name)) + return + + if len(filtered_entities) > 1: + self.log.warning(( + "Found more than one task \"{}\"" + " under parent \"{}\" in Ftrack." + ).format(task_name, asset_name)) + return + + return filtered_entities[0] + + def _ftrack_status_change(self, session, entity, project_name): + presets = config.get_presets(project_name)["ftrack"]["ftrack_config"] + statuses = presets.get("status_update") + if not statuses: + return + + actual_status = entity["status"]["name"].lower() + already_tested = set() + ent_path = "/".join( + [ent["name"] for ent in entity["link"]] + ) + while True: + next_status_name = None + for key, value in statuses.items(): + if key in already_tested: + continue + if actual_status in value or "_any_" in value: + if key != "_ignore_": + next_status_name = key + already_tested.add(key) + break + already_tested.add(key) + + if next_status_name is None: + break + + try: + query = "Status where name is \"{}\"".format( + next_status_name + ) + status = session.query(query).one() + + entity["status"] = status + session.commit() + self.log.debug("Changing status to \"{}\" <{}>".format( + next_status_name, ent_path + )) + break + + except Exception: + session.rollback() + msg = ( + "Status \"{}\" in presets wasn't found" + " on Ftrack entity type \"{}\"" + ).format(next_status_name, entity.entity_type) + self.log.warning(msg) + + def _start_timer(self, session, entity, _ftrack_api): + self.log.debug("Triggering timer start.") + + user_entity = session.query("User where username is \"{}\"".format( + os.environ["FTRACK_API_USER"] + )).first() + if not user_entity: + self.log.warning( + "Couldn't find user with username \"{}\" in Ftrack".format( + os.environ["FTRACK_API_USER"] + ) + ) + return + + source = { + "user": { + "id": user_entity["id"], + "username": user_entity["username"] + } + } + event_data = { + "actionIdentifier": "start.timer", + "selection": [{"entityId": entity["id"], "entityType": "task"}] + } + session.event_hub.publish( + _ftrack_api.event.base.Event( + topic="ftrack.action.launch", + data=event_data, + source=source + ), + on_error="ignore" + ) + self.log.debug("Timer start triggered successfully.") diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 309ea4190f0..30790651d40 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -1,18 +1,11 @@ import os -import sys import re import json import collections import logging import itertools -import copy import contextlib import subprocess -import getpass -import acre -import platform -from pype.lib.hooks import execute_hook -from .deprecated import get_avalon_database from avalon import io, pipeline import avalon.api @@ -1346,372 +1339,3 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None): ) return None return version_doc - - -class ApplicationLaunchFailed(Exception): - pass - - -def launch_application(project_name, asset_name, task_name, app_name): - database = get_avalon_database() - project_document = database[project_name].find_one({"type": "project"}) - asset_document = database[project_name].find_one({ - "type": "asset", - "name": asset_name - }) - - asset_doc_parents = asset_document["data"].get("parents") - hierarchy = "/".join(asset_doc_parents) - - app_def = avalon.lib.get_application(app_name) - app_label = app_def.get("ftrack_label", app_def.get("label", app_name)) - - host_name = app_def["application_dir"] - data = { - "project": { - "name": project_document["name"], - "code": project_document["data"].get("code") - }, - "task": task_name, - "asset": asset_name, - "app": host_name, - "hierarchy": hierarchy - } - - try: - anatomy = Anatomy(project_name) - anatomy_filled = anatomy.format(data) - workdir = os.path.normpath(anatomy_filled["work"]["folder"]) - - except Exception as exc: - raise ApplicationLaunchFailed( - "Error in anatomy.format: {}".format(str(exc)) - ) - - try: - os.makedirs(workdir) - except FileExistsError: - pass - - last_workfile_path = None - extensions = avalon.api.HOST_WORKFILE_EXTENSIONS.get(host_name) - if extensions: - # Find last workfile - file_template = anatomy.templates["work"]["file"] - data.update({ - "version": 1, - "user": os.environ.get("PYPE_USERNAME") or getpass.getuser(), - "ext": extensions[0] - }) - - last_workfile_path = avalon.api.last_workfile( - workdir, file_template, data, extensions, True - ) - - # set environments for Avalon - prep_env = copy.deepcopy(os.environ) - prep_env.update({ - "AVALON_PROJECT": project_name, - "AVALON_ASSET": asset_name, - "AVALON_TASK": task_name, - "AVALON_APP": host_name, - "AVALON_APP_NAME": app_name, - "AVALON_HIERARCHY": hierarchy, - "AVALON_WORKDIR": workdir - }) - - start_last_workfile = avalon.api.should_start_last_workfile( - project_name, host_name, task_name - ) - # Store boolean as "0"(False) or "1"(True) - prep_env["AVALON_OPEN_LAST_WORKFILE"] = ( - str(int(bool(start_last_workfile))) - ) - - if ( - start_last_workfile - and last_workfile_path - and os.path.exists(last_workfile_path) - ): - prep_env["AVALON_LAST_WORKFILE"] = last_workfile_path - - prep_env.update(anatomy.roots_obj.root_environments()) - - # collect all the 'environment' attributes from parents - tools_attr = [prep_env["AVALON_APP"], prep_env["AVALON_APP_NAME"]] - tools_env = asset_document["data"].get("tools_env") or [] - tools_attr.extend(tools_env) - - tools_env = acre.get_tools(tools_attr) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(prep_env)) - - # Get path to execute - st_temp_path = os.environ["PYPE_CONFIG"] - os_plat = platform.system().lower() - - # Path to folder with launchers - path = os.path.join(st_temp_path, "launchers", os_plat) - - # Full path to executable launcher - execfile = None - - launch_hook = app_def.get("launch_hook") - if launch_hook: - log.info("launching hook: {}".format(launch_hook)) - ret_val = execute_hook(launch_hook, env=env) - if not ret_val: - raise ApplicationLaunchFailed( - "Hook didn't finish successfully {}".format(app_label) - ) - - if sys.platform == "win32": - for ext in os.environ["PATHEXT"].split(os.pathsep): - fpath = os.path.join(path.strip('"'), app_def["executable"] + ext) - if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - execfile = fpath - break - - # Run SW if was found executable - if execfile is None: - raise ApplicationLaunchFailed( - "We didn't find launcher for {}".format(app_label) - ) - - popen = avalon.lib.launch( - executable=execfile, args=[], environment=env - ) - - elif ( - sys.platform.startswith("linux") - or sys.platform.startswith("darwin") - ): - execfile = os.path.join(path.strip('"'), app_def["executable"]) - # Run SW if was found executable - if execfile is None: - raise ApplicationLaunchFailed( - "We didn't find launcher for {}".format(app_label) - ) - - if not os.path.isfile(execfile): - raise ApplicationLaunchFailed( - "Launcher doesn't exist - {}".format(execfile) - ) - - try: - fp = open(execfile) - except PermissionError as perm_exc: - raise ApplicationLaunchFailed( - "Access denied on launcher {} - {}".format(execfile, perm_exc) - ) - - fp.close() - # check executable permission - if not os.access(execfile, os.X_OK): - raise ApplicationLaunchFailed( - "No executable permission - {}".format(execfile) - ) - - popen = avalon.lib.launch( # noqa: F841 - "/usr/bin/env", args=["bash", execfile], environment=env - ) - return popen - - -class ApplicationAction(avalon.api.Action): - """Default application launcher - - This is a convenience application Action that when "config" refers to a - parsed application `.toml` this can launch the application. - - """ - _log = None - config = None - group = None - variant = None - required_session_keys = ( - "AVALON_PROJECT", - "AVALON_ASSET", - "AVALON_TASK" - ) - - @property - def log(self): - if self._log is None: - self._log = Logger().get_logger(self.__class__.__name__) - return self._log - - def is_compatible(self, session): - for key in self.required_session_keys: - if key not in session: - return False - return True - - def process(self, session, **kwargs): - """Process the full Application action""" - - project_name = session["AVALON_PROJECT"] - asset_name = session["AVALON_ASSET"] - task_name = session["AVALON_TASK"] - launch_application( - project_name, asset_name, task_name, self.name - ) - - self._ftrack_after_launch_procedure( - project_name, asset_name, task_name - ) - - def _ftrack_after_launch_procedure( - self, project_name, asset_name, task_name - ): - # TODO move to launch hook - required_keys = ("FTRACK_SERVER", "FTRACK_API_USER", "FTRACK_API_KEY") - for key in required_keys: - if not os.environ.get(key): - self.log.debug(( - "Missing required environment \"{}\"" - " for Ftrack after launch procedure." - ).format(key)) - return - - try: - import ftrack_api - session = ftrack_api.Session(auto_connect_event_hub=True) - self.log.debug("Ftrack session created") - except Exception: - self.log.warning("Couldn't create Ftrack session") - return - - try: - entity = self._find_ftrack_task_entity( - session, project_name, asset_name, task_name - ) - self._ftrack_status_change(session, entity, project_name) - self._start_timer(session, entity, ftrack_api) - except Exception: - self.log.warning( - "Couldn't finish Ftrack procedure.", exc_info=True - ) - return - - finally: - session.close() - - def _find_ftrack_task_entity( - self, session, project_name, asset_name, task_name - ): - project_entity = session.query( - "Project where full_name is \"{}\"".format(project_name) - ).first() - if not project_entity: - self.log.warning( - "Couldn't find project \"{}\" in Ftrack.".format(project_name) - ) - return - - potential_task_entities = session.query(( - "TypedContext where parent.name is \"{}\" and project_id is \"{}\"" - ).format(asset_name, project_entity["id"])).all() - filtered_entities = [] - for _entity in potential_task_entities: - if ( - _entity.entity_type.lower() == "task" - and _entity["name"] == task_name - ): - filtered_entities.append(_entity) - - if not filtered_entities: - self.log.warning(( - "Couldn't find task \"{}\" under parent \"{}\" in Ftrack." - ).format(task_name, asset_name)) - return - - if len(filtered_entities) > 1: - self.log.warning(( - "Found more than one task \"{}\"" - " under parent \"{}\" in Ftrack." - ).format(task_name, asset_name)) - return - - return filtered_entities[0] - - def _ftrack_status_change(self, session, entity, project_name): - presets = config.get_presets(project_name)["ftrack"]["ftrack_config"] - statuses = presets.get("status_update") - if not statuses: - return - - actual_status = entity["status"]["name"].lower() - already_tested = set() - ent_path = "/".join( - [ent["name"] for ent in entity["link"]] - ) - while True: - next_status_name = None - for key, value in statuses.items(): - if key in already_tested: - continue - if actual_status in value or "_any_" in value: - if key != "_ignore_": - next_status_name = key - already_tested.add(key) - break - already_tested.add(key) - - if next_status_name is None: - break - - try: - query = "Status where name is \"{}\"".format( - next_status_name - ) - status = session.query(query).one() - - entity["status"] = status - session.commit() - self.log.debug("Changing status to \"{}\" <{}>".format( - next_status_name, ent_path - )) - break - - except Exception: - session.rollback() - msg = ( - "Status \"{}\" in presets wasn't found" - " on Ftrack entity type \"{}\"" - ).format(next_status_name, entity.entity_type) - self.log.warning(msg) - - def _start_timer(self, session, entity, _ftrack_api): - self.log.debug("Triggering timer start.") - - user_entity = session.query("User where username is \"{}\"".format( - os.environ["FTRACK_API_USER"] - )).first() - if not user_entity: - self.log.warning( - "Couldn't find user with username \"{}\" in Ftrack".format( - os.environ["FTRACK_API_USER"] - ) - ) - return - - source = { - "user": { - "id": user_entity["id"], - "username": user_entity["username"] - } - } - event_data = { - "actionIdentifier": "start.timer", - "selection": [{"entityId": entity["id"], "entityType": "task"}] - } - session.event_hub.publish( - _ftrack_api.event.base.Event( - topic="ftrack.action.launch", - data=event_data, - source=source - ), - on_error="ignore" - ) - self.log.debug("Timer start triggered successfully.") From c9b0ab8bd9b0231fea4e5adc24bbcff2e1cccb9b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:21:59 +0100 Subject: [PATCH 15/47] added few comments --- pype/lib/applications.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/lib/applications.py b/pype/lib/applications.py index f4861ae8668..fd3d0ef9906 100644 --- a/pype/lib/applications.py +++ b/pype/lib/applications.py @@ -21,6 +21,11 @@ class ApplicationLaunchFailed(Exception): def launch_application(project_name, asset_name, task_name, app_name): + """Launch host application with filling required environments. + + TODO(iLLiCiT): This should be split into more parts. + """ + # `get_avalon_database` is in Pype 3 replaced with using `AvalonMongoDB` database = get_avalon_database() project_document = database[project_name].find_one({"type": "project"}) asset_document = database[project_name].find_one({ @@ -35,6 +40,7 @@ def launch_application(project_name, asset_name, task_name, app_name): app_label = app_def.get("ftrack_label", app_def.get("label", app_name)) host_name = app_def["application_dir"] + # Workfile data collection may be special function? data = { "project": { "name": project_document["name"], From b407291401897f47a7153822e6e3b0a2532b259b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:23:03 +0100 Subject: [PATCH 16/47] removed empoty lines --- pype/lib/lib_old.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 30790651d40..421e3204150 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -523,10 +523,6 @@ def get_last_version_from_path(path_dir, filter): return None - - - - def get_subsets(asset_name, regex_filter=None, version=None, From 83c4696591918af18c25393c697c0fcb2ae69693 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 11:55:25 +0100 Subject: [PATCH 17/47] moved `map_subsets_by_family` under `BuildWorkfile` class --- pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 30 +++++++++++++++--------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 6620bd5a122..a303bf038dc 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -36,7 +36,6 @@ get_last_version_from_path, get_subsets, get_linked_assets, - map_subsets_by_family, BuildWorkfile, ffprobe_streams, source_hash, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 421e3204150..b384c3a06a3 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -608,20 +608,6 @@ def get_linked_assets(asset_entity): return inputs -def map_subsets_by_family(subsets): - subsets_by_family = collections.defaultdict(list) - for subset in subsets: - family = subset["data"].get("family") - if not family: - families = subset["data"].get("families") - if not families: - continue - family = families[0] - - subsets_by_family[family].append(subset) - return subsets_by_family - - class BuildWorkfile: """Wrapper for build workfile process. @@ -629,6 +615,20 @@ class BuildWorkfile: are host related, since each host has it's loaders. """ + @staticmethod + def map_subsets_by_family(subsets): + subsets_by_family = collections.defaultdict(list) + for subset in subsets: + family = subset["data"].get("family") + if not family: + families = subset["data"].get("families") + if not families: + continue + family = families[0] + + subsets_by_family[family].append(subset) + return subsets_by_family + def process(self): """Main method of this wrapper. @@ -901,7 +901,7 @@ def _prepare_profile_for_subsets(self, subsets, profiles): :rtype: dict """ # Prepare subsets - subsets_by_family = map_subsets_by_family(subsets) + subsets_by_family = self.map_subsets_by_family(subsets) profiles_per_subset_id = {} for family, subsets in subsets_by_family.items(): From 389d4756e94db84de83d21e08d391ccaf4ed8bd7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 4 Nov 2020 16:19:11 +0100 Subject: [PATCH 18/47] Import tests for #669 --- pype/tests/README.md | 4 ++++ pype/tests/test_lib_restructuralization.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 pype/tests/README.md create mode 100644 pype/tests/test_lib_restructuralization.py diff --git a/pype/tests/README.md b/pype/tests/README.md new file mode 100644 index 00000000000..c05166767c4 --- /dev/null +++ b/pype/tests/README.md @@ -0,0 +1,4 @@ +Tests for Pype +-------------- +Trigger by: + `pype test --pype` \ No newline at end of file diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py new file mode 100644 index 00000000000..e167c5f5553 --- /dev/null +++ b/pype/tests/test_lib_restructuralization.py @@ -0,0 +1,13 @@ +# Test for backward compability of restructure of lib.py into lib library +# #664 +# Contains simple imports that should still work + + +def test_backward_compatibility(printer): + printer("Test if imports still work") + try: + from pype.lib import filter_pyblish_plugins + from pype.lib import execute_hook + from pype.lib import PypeHook + except ImportError as e: + raise \ No newline at end of file From 3ac01c55432d6e9d8d3b58810e0f64540399ba3f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 4 Nov 2020 18:49:09 +0100 Subject: [PATCH 19/47] Import tests for #681 --- pype/tests/test_lib_restructuralization.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py index e167c5f5553..92197c82326 100644 --- a/pype/tests/test_lib_restructuralization.py +++ b/pype/tests/test_lib_restructuralization.py @@ -1,5 +1,4 @@ -# Test for backward compability of restructure of lib.py into lib library -# #664 +# Test for backward compatibility of restructure of lib.py into lib library # Contains simple imports that should still work @@ -9,5 +8,13 @@ def test_backward_compatibility(printer): from pype.lib import filter_pyblish_plugins from pype.lib import execute_hook from pype.lib import PypeHook + + from pype.lib import get_latest_version + from pype.lib import ApplicationLaunchFailed + from pype.lib import launch_application + from pype.lib import ApplicationAction + from pype.lib import get_avalon_database + from pype.lib import set_io_database + except ImportError as e: - raise \ No newline at end of file + raise From 8d74d69a016c32260dabae5e2e419ed5f071badb Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:02:11 +0100 Subject: [PATCH 20/47] move grouper to extractor --- pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 12 ------------ .../maya/publish/extract_camera_mayaScene.py | 13 ++++++++++++- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index a303bf038dc..06960f5ddb3 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -24,7 +24,6 @@ add_tool_to_environment, modified_environ, pairwise, - grouper, is_latest, any_outdated, _rreplace, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index b384c3a06a3..dfb9aed6b9d 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -235,18 +235,6 @@ def pairwise(iterable): return itertools.izip(a, a) -def grouper(iterable, n, fillvalue=None): - """Collect data into fixed-length chunks or blocks - - Examples: - grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx - - """ - - args = [iter(iterable)] * n - return itertools.izip_longest(fillvalue=fillvalue, *args) - - def is_latest(representation): """Return whether the representation is from latest version diff --git a/pype/plugins/maya/publish/extract_camera_mayaScene.py b/pype/plugins/maya/publish/extract_camera_mayaScene.py index 1a0f4694d1d..65c5ef5840c 100644 --- a/pype/plugins/maya/publish/extract_camera_mayaScene.py +++ b/pype/plugins/maya/publish/extract_camera_mayaScene.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- """Extract camera as Maya Scene.""" import os +import itertools from maya import cmds import avalon.maya import pype.api -from pype.lib import grouper from pype.hosts.maya import lib @@ -36,6 +36,17 @@ def massage_ma_file(path): f.close() +def grouper(iterable, n, fillvalue=None): + """Collect data into fixed-length chunks or blocks. + + Examples: + grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx + + """ + args = [iter(iterable)] * n + return itertools.izip_longest(fillvalue=fillvalue, *args) + + def unlock(plug): """Unlocks attribute and disconnects inputs for a plug. From ba66d038aa8434e8c737cb99a1165f4654c6176e Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:02:42 +0100 Subject: [PATCH 21/47] move pairwise to plugin --- pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 6 ------ pype/plugins/maya/publish/collect_yeti_rig.py | 7 +++---- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 06960f5ddb3..18531ca05c5 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -23,7 +23,6 @@ get_hierarchy, add_tool_to_environment, modified_environ, - pairwise, is_latest, any_outdated, _rreplace, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index dfb9aed6b9d..7f5523b6b55 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -229,12 +229,6 @@ def modified_environ(*remove, **update): [env.pop(k) for k in remove_after] -def pairwise(iterable): - """s -> (s0,s1), (s2,s3), (s4, s5), ...""" - a = iter(iterable) - return itertools.izip(a, a) - - def is_latest(representation): """Return whether the representation is from latest version diff --git a/pype/plugins/maya/publish/collect_yeti_rig.py b/pype/plugins/maya/publish/collect_yeti_rig.py index 8a7971f3ae5..73a84b00736 100644 --- a/pype/plugins/maya/publish/collect_yeti_rig.py +++ b/pype/plugins/maya/publish/collect_yeti_rig.py @@ -6,7 +6,6 @@ import pyblish.api from pype.hosts.maya import lib -from pype.lib import pairwise SETTINGS = {"renderDensity", @@ -78,7 +77,7 @@ def collect_input_connections(self, instance): connections = cmds.ls(connections, long=True) # Ensure long names inputs = [] - for dest, src in pairwise(connections): + for dest, src in lib.pairwise(connections): source_node, source_attr = src.split(".", 1) dest_node, dest_attr = dest.split(".", 1) @@ -119,7 +118,7 @@ def get_yeti_resources(self, node): texture_filenames = [] if image_search_paths: - + # TODO: Somehow this uses OS environment path separator, `:` vs `;` # Later on check whether this is pipeline OS cross-compatible. image_search_paths = [p for p in @@ -127,7 +126,7 @@ def get_yeti_resources(self, node): # find all ${TOKEN} tokens and replace them with $TOKEN env. variable image_search_paths = self._replace_tokens(image_search_paths) - + # List all related textures texture_filenames = cmds.pgYetiCommand(node, listTextures=True) self.log.info("Found %i texture(s)" % len(texture_filenames)) From e86215b1426ccbc07cde4aba4f1ab2eed0f023ed Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:02:57 +0100 Subject: [PATCH 22/47] move pairwise to maya.lib --- pype/hosts/maya/lib.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pype/hosts/maya/lib.py b/pype/hosts/maya/lib.py index e7ca5ec4dc5..fcb627493df 100644 --- a/pype/hosts/maya/lib.py +++ b/pype/hosts/maya/lib.py @@ -122,6 +122,12 @@ def float_round(num, places=0, direction=ceil): return direction(num * (10**places)) / float(10**places) +def pairwise(iterable): + """s -> (s0,s1), (s2,s3), (s4, s5), ...""" + a = iter(iterable) + return itertools.izip(a, a) + + def unique(name): assert isinstance(name, string_types), "`name` must be string" @@ -419,12 +425,12 @@ def empty_sets(sets, force=False): plugs=True, connections=True) or [] original_connections.extend(connections) - for dest, src in lib.pairwise(connections): + for dest, src in pairwise(connections): cmds.disconnectAttr(src, dest) yield finally: - for dest, src in lib.pairwise(original_connections): + for dest, src in pairwise(original_connections): cmds.connectAttr(src, dest) # Restore original members From 1ad1cea494b435b5b66fc9b28909b58dd0fa7ff7 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:04:22 +0100 Subject: [PATCH 23/47] remove modified_environ function --- pype/api.py | 1 - pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 32 ------- .../launcher/actions/unused/PremierePro.py | 83 ------------------- 4 files changed, 117 deletions(-) delete mode 100644 pype/plugins/launcher/actions/unused/PremierePro.py diff --git a/pype/api.py b/pype/api.py index 2c7dfa73f07..a6762beca34 100644 --- a/pype/api.py +++ b/pype/api.py @@ -43,7 +43,6 @@ get_subsets, get_version_from_path, get_last_version_from_path, - modified_environ, add_tool_to_environment, source_hash, get_latest_version diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 18531ca05c5..c9ea8e76764 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -22,7 +22,6 @@ get_ffmpeg_tool_path, get_hierarchy, add_tool_to_environment, - modified_environ, is_latest, any_outdated, _rreplace, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 7f5523b6b55..92ce206bf7d 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -197,38 +197,6 @@ def add_tool_to_environment(tools): os.environ.update(env) -@contextlib.contextmanager -def modified_environ(*remove, **update): - """ - Temporarily updates the ``os.environ`` dictionary in-place. - - The ``os.environ`` dictionary is updated in-place so that the modification - is sure to work in all situations. - - :param remove: Environment variables to remove. - :param update: Dictionary of environment variables - and values to add/update. - """ - env = os.environ - update = update or {} - remove = remove or [] - - # List of environment variables being updated or removed. - stomped = (set(update.keys()) | set(remove)) & set(env.keys()) - # Environment variables and values to restore on exit. - update_after = {k: env[k] for k in stomped} - # Environment variables and values to remove on exit. - remove_after = frozenset(k for k in update if k not in env) - - try: - env.update(update) - [env.pop(k, None) for k in remove] - yield - finally: - env.update(update_after) - [env.pop(k) for k in remove_after] - - def is_latest(representation): """Return whether the representation is from latest version diff --git a/pype/plugins/launcher/actions/unused/PremierePro.py b/pype/plugins/launcher/actions/unused/PremierePro.py deleted file mode 100644 index e460af14515..00000000000 --- a/pype/plugins/launcher/actions/unused/PremierePro.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import acre - -from avalon import api, lib, io -import pype.api as pype - - -class PremierePro(api.Action): - - name = "premiere_2019" - label = "Premiere Pro" - icon = "premiere_icon" - order = 996 - - def is_compatible(self, session): - """Return whether the action is compatible with the session""" - if "AVALON_TASK" in session: - return True - return False - - def process(self, session, **kwargs): - """Implement the behavior for when the action is triggered - - Args: - session (dict): environment dictionary - - Returns: - Popen instance of newly spawned process - - """ - - with pype.modified_environ(**session): - # Get executable by name - app = lib.get_application(self.name) - executable = lib.which(app["executable"]) - - # Run as server - arguments = [] - - tools_env = acre.get_tools([self.name]) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(os.environ)) - - if not env.get('AVALON_WORKDIR', None): - project_name = env.get("AVALON_PROJECT") - anatomy = pype.Anatomy(project_name) - os.environ['AVALON_PROJECT'] = project_name - io.Session['AVALON_PROJECT'] = project_name - - task_name = os.environ.get( - "AVALON_TASK", io.Session["AVALON_TASK"] - ) - asset_name = os.environ.get( - "AVALON_ASSET", io.Session["AVALON_ASSET"] - ) - application = lib.get_application( - os.environ["AVALON_APP_NAME"] - ) - - project_doc = io.find_one({"type": "project"}) - data = { - "task": task_name, - "asset": asset_name, - "project": { - "name": project_doc["name"], - "code": project_doc["data"].get("code", '') - }, - "hierarchy": pype.get_hierarchy(), - "app": application["application_dir"] - } - anatomy_filled = anatomy.format(data) - workdir = anatomy_filled["work"]["folder"] - - os.environ["AVALON_WORKDIR"] = workdir - - env.update(dict(os.environ)) - - lib.launch( - executable=executable, - args=arguments, - environment=env - ) - return From dca965b6915af50c910818c27701924f7e3c0719 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:05:32 +0100 Subject: [PATCH 24/47] remove _get_host --- pype/lib/__init__.py | 1 - pype/lib/lib_old.py | 7 ------- 2 files changed, 8 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index c9ea8e76764..7c9c7b0d413 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -27,7 +27,6 @@ _rreplace, version_up, switch_item, - _get_host_name, get_asset, get_version_from_path, get_last_version_from_path, diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 92ce206bf7d..cbece671f57 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -391,13 +391,6 @@ def switch_item(container, return representation -def _get_host_name(): - - _host = avalon.api.registered_host() - # This covers nested module name like avalon.maya - return _host.__name__.rsplit(".", 1)[-1] - - def get_asset(asset_name=None): """ Returning asset document from database """ if not asset_name: From 63b32be427fafe82edaac01743271259393b3613 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:05:51 +0100 Subject: [PATCH 25/47] move version and path related functions to new lib --- pype/api.py | 1 - pype/lib/__init__.py | 23 +++-- pype/lib/lib_old.py | 211 ++--------------------------------------- pype/lib/path_tools.py | 172 +++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+), 210 deletions(-) create mode 100644 pype/lib/path_tools.py diff --git a/pype/api.py b/pype/api.py index a6762beca34..d5f0ced1a80 100644 --- a/pype/api.py +++ b/pype/api.py @@ -43,7 +43,6 @@ get_subsets, get_version_from_path, get_last_version_from_path, - add_tool_to_environment, source_hash, get_latest_version ) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 7c9c7b0d413..05aee8cf7b6 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -16,20 +16,21 @@ from .plugin_tools import filter_pyblish_plugins +from .path_tools import ( + version_up, + get_version_from_path, + get_last_version_from_path, + get_paths_from_environ, + get_ffmpeg_tool_path +) + from .lib_old import ( _subprocess, - get_paths_from_environ, - get_ffmpeg_tool_path, get_hierarchy, - add_tool_to_environment, is_latest, any_outdated, - _rreplace, - version_up, switch_item, get_asset, - get_version_from_path, - get_last_version_from_path, get_subsets, get_linked_assets, BuildWorkfile, @@ -49,5 +50,11 @@ "launch_application", "ApplicationAction", - "filter_pyblish_plugins" + "filter_pyblish_plugins", + + "version_up", + "get_version_from_path", + "get_last_version_from_path", + "get_paths_from_environ", + "get_ffmpeg_tool_path" ] diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index cbece671f57..eafd34264c4 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -14,62 +14,6 @@ log = logging.getLogger(__name__) -def get_paths_from_environ(env_key, return_first=False): - """Return existing paths from specific envirnment variable. - - :param env_key: Environment key where should look for paths. - :type env_key: str - :param return_first: Return first path on `True`, list of all on `False`. - :type return_first: boolean - - Difference when none of paths exists: - - when `return_first` is set to `False` then function returns empty list. - - when `return_first` is set to `True` then function returns `None`. - """ - - existing_paths = [] - paths = os.environ.get(env_key) or "" - path_items = paths.split(os.pathsep) - for path in path_items: - # Skip empty string - if not path: - continue - # Normalize path - path = os.path.normpath(path) - # Check if path exists - if os.path.exists(path): - # Return path if `return_first` is set to True - if return_first: - return path - # Store path - existing_paths.append(path) - - # Return None if none of paths exists - if return_first: - return None - # Return all existing paths from environment variable - return existing_paths - - -def get_ffmpeg_tool_path(tool="ffmpeg"): - """Find path to ffmpeg tool in FFMPEG_PATH paths. - - Function looks for tool in paths set in FFMPEG_PATH environment. If tool - exists then returns it's full path. - - Returns tool name itself when tool path was not found. (FFmpeg path may be - set in PATH environment variable) - """ - - dir_paths = get_paths_from_environ("FFMPEG_PATH") - for dir_path in dir_paths: - for file_name in os.listdir(dir_path): - base, ext = os.path.splitext(file_name) - if base.lower() == tool.lower(): - return os.path.join(dir_path, tool) - return tool - - # Special naming case for subprocess since its a built-in method. def _subprocess(*args, **kwargs): """Convenience method for getting output errors for subprocess. @@ -135,9 +79,11 @@ def _subprocess(*args, **kwargs): return full_output +# Avalon databse functions + + def get_hierarchy(asset_name=None): - """ - Obtain asset hierarchy path string from mongo db + """Obtain asset hierarchy path string from mongo db. Returns: string: asset hierarchy path @@ -179,26 +125,8 @@ def get_hierarchy(asset_name=None): return "/".join(hierarchy_items) -def add_tool_to_environment(tools): - """ - It is adding dynamic environment to os environment. - - Args: - tool (list, tuple): list of tools, name should corespond to json/toml - - Returns: - os.environ[KEY]: adding to os.environ - """ - - import acre - tools_env = acre.get_tools(tools) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(os.environ)) - os.environ.update(env) - - def is_latest(representation): - """Return whether the representation is from latest version + """Return whether the representation is from latest version. Args: representation (dict): The representation document from the database. @@ -207,7 +135,6 @@ def is_latest(representation): bool: Whether the representation is of latest version. """ - version = io.find_one({"_id": representation['parent']}) if version["type"] == "master_version": return True @@ -225,8 +152,7 @@ def is_latest(representation): def any_outdated(): - """Return whether the current scene has any outdated content""" - + """Return whether the current scene has any outdated content.""" checked = set() host = avalon.api.registered_host() for container in host.ls(): @@ -243,73 +169,15 @@ def any_outdated(): ) if representation_doc and not is_latest(representation_doc): return True - elif not representation_doc: - log.debug("Container '{objectName}' has an invalid " - "representation, it is missing in the " - "database".format(**container)) + + log.debug("Container '{objectName}' has an invalid " + "representation, it is missing in the " + "database".format(**container)) checked.add(representation) return False -def _rreplace(s, a, b, n=1): - """Replace a with b in string s from right side n times""" - return b.join(s.rsplit(a, n)) - - -def version_up(filepath): - """Version up filepath to a new non-existing version. - - Parses for a version identifier like `_v001` or `.v001` - When no version present _v001 is appended as suffix. - - Returns: - str: filepath with increased version number - - """ - - dirname = os.path.dirname(filepath) - basename, ext = os.path.splitext(os.path.basename(filepath)) - - regex = r"[._]v\d+" - matches = re.findall(regex, str(basename), re.IGNORECASE) - if not matches: - log.info("Creating version...") - new_label = "_v{version:03d}".format(version=1) - new_basename = "{}{}".format(basename, new_label) - else: - label = matches[-1] - version = re.search(r"\d+", label).group() - padding = len(version) - - new_version = int(version) + 1 - new_version = '{version:0{padding}d}'.format(version=new_version, - padding=padding) - new_label = label.replace(version, new_version, 1) - new_basename = _rreplace(basename, label, new_label) - - if not new_basename.endswith(new_label): - index = (new_basename.find(new_label)) - index += len(new_label) - new_basename = new_basename[:index] - - new_filename = "{}{}".format(new_basename, ext) - new_filename = os.path.join(dirname, new_filename) - new_filename = os.path.normpath(new_filename) - - if new_filename == filepath: - raise RuntimeError("Created path is the same as current file," - "this is a bug") - - for file in os.listdir(dirname): - if file.endswith(ext) and file.startswith(new_basename): - log.info("Skipping existing version %s" % new_label) - return version_up(new_filename) - - log.info("New version %s" % new_label) - return new_filename - - def switch_item(container, asset_name=None, subset_name=None, @@ -407,65 +275,6 @@ def get_asset(asset_name=None): return asset_document -def get_version_from_path(file): - """ - Finds version number in file path string - - Args: - file (string): file path - - Returns: - v: version number in string ('001') - - """ - pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE) - try: - return pattern.findall(file)[0] - except IndexError: - log.error( - "templates:get_version_from_workfile:" - "`{}` missing version string." - "Example `v004`".format(file) - ) - - -def get_last_version_from_path(path_dir, filter): - """ - Finds last version of given directory content - - Args: - path_dir (string): directory path - filter (list): list of strings used as file name filter - - Returns: - string: file name with last version - - Example: - last_version_file = get_last_version_from_path( - "/project/shots/shot01/work", ["shot01", "compositing", "nk"]) - """ - - assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory" - assert isinstance(filter, list) and ( - len(filter) != 0), "`filter` argument needs to be list and not empty" - - filtred_files = list() - - # form regex for filtering - patern = r".*".join(filter) - - for f in os.listdir(path_dir): - if not re.findall(patern, f): - continue - filtred_files.append(f) - - if filtred_files: - sorted(filtred_files) - return filtred_files[-1] - else: - return None - - def get_subsets(asset_name, regex_filter=None, version=None, diff --git a/pype/lib/path_tools.py b/pype/lib/path_tools.py new file mode 100644 index 00000000000..c56f4c8974c --- /dev/null +++ b/pype/lib/path_tools.py @@ -0,0 +1,172 @@ +import os +import re +import logging + +log = logging.getLogger(__name__) + + +def get_paths_from_environ(env_key, return_first=False): + """Return existing paths from specific envirnment variable. + + :param env_key: Environment key where should look for paths. + :type env_key: str + :param return_first: Return first path on `True`, list of all on `False`. + :type return_first: boolean + + Difference when none of paths exists: + - when `return_first` is set to `False` then function returns empty list. + - when `return_first` is set to `True` then function returns `None`. + """ + existing_paths = [] + paths = os.environ.get(env_key) or "" + path_items = paths.split(os.pathsep) + for path in path_items: + # Skip empty string + if not path: + continue + # Normalize path + path = os.path.normpath(path) + # Check if path exists + if os.path.exists(path): + # Return path if `return_first` is set to True + if return_first: + return path + # Store path + existing_paths.append(path) + + # Return None if none of paths exists + if return_first: + return None + # Return all existing paths from environment variable + return existing_paths + + +def get_ffmpeg_tool_path(tool="ffmpeg"): + """Find path to ffmpeg tool in FFMPEG_PATH paths. + + Function looks for tool in paths set in FFMPEG_PATH environment. If tool + exists then returns it's full path. + + Returns tool name itself when tool path was not found. (FFmpeg path may be + set in PATH environment variable) + """ + dir_paths = get_paths_from_environ("FFMPEG_PATH") + for dir_path in dir_paths: + for file_name in os.listdir(dir_path): + base, _ext = os.path.splitext(file_name) + if base.lower() == tool.lower(): + return os.path.join(dir_path, tool) + return tool + + +def _rreplace(s, a, b, n=1): + """Replace a with b in string s from right side n times.""" + return b.join(s.rsplit(a, n)) + + +def version_up(filepath): + """Version up filepath to a new non-existing version. + + Parses for a version identifier like `_v001` or `.v001` + When no version present _v001 is appended as suffix. + + Returns: + str: filepath with increased version number + + """ + dirname = os.path.dirname(filepath) + basename, ext = os.path.splitext(os.path.basename(filepath)) + + regex = r"[._]v\d+" + matches = re.findall(regex, str(basename), re.IGNORECASE) + if not matches: + log.info("Creating version...") + new_label = "_v{version:03d}".format(version=1) + new_basename = "{}{}".format(basename, new_label) + else: + label = matches[-1] + version = re.search(r"\d+", label).group() + padding = len(version) + + new_version = int(version) + 1 + new_version = '{version:0{padding}d}'.format(version=new_version, + padding=padding) + new_label = label.replace(version, new_version, 1) + new_basename = _rreplace(basename, label, new_label) + + if not new_basename.endswith(new_label): + index = (new_basename.find(new_label)) + index += len(new_label) + new_basename = new_basename[:index] + + new_filename = "{}{}".format(new_basename, ext) + new_filename = os.path.join(dirname, new_filename) + new_filename = os.path.normpath(new_filename) + + if new_filename == filepath: + raise RuntimeError("Created path is the same as current file," + "this is a bug") + + for file in os.listdir(dirname): + if file.endswith(ext) and file.startswith(new_basename): + log.info("Skipping existing version %s" % new_label) + return version_up(new_filename) + + log.info("New version %s" % new_label) + return new_filename + + +def get_version_from_path(file): + """Find version number in file path string.s + + Args: + file (string): file path + + Returns: + v: version number in string ('001') + + """ + pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE) + try: + return pattern.findall(file)[0] + except IndexError: + log.error( + "templates:get_version_from_workfile:" + "`{}` missing version string." + "Example `v004`".format(file) + ) + + +def get_last_version_from_path(path_dir, filter): + """Find last version of given directory content. + + Args: + path_dir (string): directory path + filter (list): list of strings used as file name filter + + Returns: + string: file name with last version + + Example: + last_version_file = get_last_version_from_path( + "/project/shots/shot01/work", ["shot01", "compositing", "nk"]) + """ + assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory" + assert isinstance(filter, list) and ( + len(filter) != 0), "`filter` argument needs to be list and not empty" + + filtred_files = list() + + # form regex for filtering + patern = r".*".join(filter) + + for file in os.listdir(path_dir): + if not re.findall(patern, file): + continue + filtred_files.append(file) + + if filtred_files: + sorted(filtred_files) + return filtred_files[-1] + + return None From 5b891e06cf34489356fa05cb1a1d622cf7e2b225 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:07:11 +0100 Subject: [PATCH 26/47] typo in dosctring --- pype/lib/path_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/lib/path_tools.py b/pype/lib/path_tools.py index c56f4c8974c..11fc3bf8a64 100644 --- a/pype/lib/path_tools.py +++ b/pype/lib/path_tools.py @@ -117,7 +117,7 @@ def version_up(filepath): def get_version_from_path(file): - """Find version number in file path string.s + """Find version number in file path string. Args: file (string): file path From 6fb10dba1f9ba0ef6f6b92025728d5495b16b479 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:18:07 +0100 Subject: [PATCH 27/47] add missing import --- pype/hosts/maya/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/hosts/maya/lib.py b/pype/hosts/maya/lib.py index fcb627493df..37e4ccf9150 100644 --- a/pype/hosts/maya/lib.py +++ b/pype/hosts/maya/lib.py @@ -8,6 +8,7 @@ import bson import json import logging +import itertools import contextlib from collections import OrderedDict, defaultdict from math import ceil From 28ad05fb7b6bda1ed9bc54719bb7d5b11d9dd189 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 10 Nov 2020 01:22:20 +0100 Subject: [PATCH 28/47] remvoe deleted function from API --- pype/api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/api.py b/pype/api.py index d5f0ced1a80..37ddef89724 100644 --- a/pype/api.py +++ b/pype/api.py @@ -90,8 +90,6 @@ "get_subsets", "get_version_from_path", "get_last_version_from_path", - "modified_environ", - "add_tool_to_environment", "source_hash", "subprocess", From da8f010c9062b12547e9c0191e3d0453a156cef5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 10:10:26 +0100 Subject: [PATCH 29/47] updated lib_old with newer changes --- pype/lib/lib_old.py | 70 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index b384c3a06a3..114996cd90c 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -9,7 +9,7 @@ from avalon import io, pipeline import avalon.api -from ..api import config, Anatomy, Logger +from ..api import config log = logging.getLogger(__name__) @@ -695,10 +695,10 @@ def build_workfile(self): current_task_name = io.Session["AVALON_TASK"] # Load workfile presets for task - build_presets = self.get_build_presets(current_task_name) + self.build_presets = self.get_build_presets(current_task_name) # Skip if there are any presets for task - if not build_presets: + if not self.build_presets: log.warning( "Current task `{}` does not have any loading preset.".format( current_task_name @@ -707,9 +707,9 @@ def build_workfile(self): return # Get presets for loading current asset - current_context_profiles = build_presets.get("current_context") + current_context_profiles = self.build_presets.get("current_context") # Get presets for loading linked assets - link_context_profiles = build_presets.get("linked_assets") + link_context_profiles = self.build_presets.get("linked_assets") # Skip if both are missing if not current_context_profiles and not link_context_profiles: log.warning("Current task `{}` has empty loading preset.".format( @@ -901,7 +901,7 @@ def _prepare_profile_for_subsets(self, subsets, profiles): :rtype: dict """ # Prepare subsets - subsets_by_family = self.map_subsets_by_family(subsets) + subsets_by_family = map_subsets_by_family(subsets) profiles_per_subset_id = {} for family, subsets in subsets_by_family.items(): @@ -1062,7 +1062,36 @@ def _load_containers( :rtype: list """ loaded_containers = [] - for subset_id, repres in repres_by_subset_id.items(): + + # Get subset id order from build presets. + build_presets = self.build_presets.get("current_context", []) + build_presets += self.build_presets.get("linked_assets", []) + subset_ids_ordered = [] + for preset in build_presets: + for preset_family in preset["families"]: + for id, subset in subsets_by_id.items(): + if preset_family not in subset["data"].get("families", []): + continue + + subset_ids_ordered.append(id) + + # Order representations from subsets. + print("repres_by_subset_id", repres_by_subset_id) + representations_ordered = [] + representations = [] + for id in subset_ids_ordered: + for subset_id, repres in repres_by_subset_id.items(): + if repres in representations: + continue + + if id == subset_id: + representations_ordered.append((subset_id, repres)) + representations.append(repres) + + print("representations", representations) + + # Load ordered reprensentations. + for subset_id, repres in representations_ordered: subset_name = subsets_by_id[subset_id]["name"] profile = profiles_per_subset_id[subset_id] @@ -1222,13 +1251,15 @@ def _collect_last_version_repres(self, asset_entities): return output -def ffprobe_streams(path_to_file): +def ffprobe_streams(path_to_file, logger=None): """Load streams from entered filepath via ffprobe.""" - log.info( + if not logger: + logger = log + logger.info( "Getting information about input \"{}\".".format(path_to_file) ) args = [ - get_ffmpeg_tool_path("ffprobe"), + "\"{}\"".format(get_ffmpeg_tool_path("ffprobe")), "-v quiet", "-print_format json", "-show_format", @@ -1236,12 +1267,21 @@ def ffprobe_streams(path_to_file): "\"{}\"".format(path_to_file) ] command = " ".join(args) - log.debug("FFprobe command: \"{}\"".format(command)) - popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + logger.debug("FFprobe command: \"{}\"".format(command)) + popen = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + popen_stdout, popen_stderr = popen.communicate() + if popen_stdout: + logger.debug("ffprobe stdout: {}".format(popen_stdout)) - popen_output = popen.communicate()[0] - log.debug("FFprobe output: {}".format(popen_output)) - return json.loads(popen_output)["streams"] + if popen_stderr: + logger.debug("ffprobe stderr: {}".format(popen_stderr)) + return json.loads(popen_stdout)["streams"] def source_hash(filepath, *args): From 324636a490e273ede976f4e78753913e039d202c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 10:17:59 +0100 Subject: [PATCH 30/47] fix method `map_subsets_by_family` call --- pype/lib/lib_old.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 114996cd90c..33ddf9ed498 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -901,7 +901,7 @@ def _prepare_profile_for_subsets(self, subsets, profiles): :rtype: dict """ # Prepare subsets - subsets_by_family = map_subsets_by_family(subsets) + subsets_by_family = self.map_subsets_by_family(subsets) profiles_per_subset_id = {} for family, subsets in subsets_by_family.items(): From 4be779b9f69210f976c17b943699b8eed7a1ded7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 10:25:06 +0100 Subject: [PATCH 31/47] BuildWorkfile class has it's own logger --- pype/lib/lib_old.py | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 33ddf9ed498..1f608e368e3 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -615,6 +615,8 @@ class BuildWorkfile: are host related, since each host has it's loaders. """ + log = logging.getLogger("BuildWorkfile") + @staticmethod def map_subsets_by_family(subsets): subsets_by_family = collections.defaultdict(list) @@ -688,7 +690,7 @@ def build_workfile(self): # Skip if there are any loaders if not loaders_by_name: - log.warning("There are no registered loaders.") + self.log.warning("There are no registered loaders.") return # Get current task name @@ -699,7 +701,7 @@ def build_workfile(self): # Skip if there are any presets for task if not self.build_presets: - log.warning( + self.log.warning( "Current task `{}` does not have any loading preset.".format( current_task_name ) @@ -712,19 +714,21 @@ def build_workfile(self): link_context_profiles = self.build_presets.get("linked_assets") # Skip if both are missing if not current_context_profiles and not link_context_profiles: - log.warning("Current task `{}` has empty loading preset.".format( - current_task_name - )) + self.log.warning( + "Current task `{}` has empty loading preset.".format( + current_task_name + ) + ) return elif not current_context_profiles: - log.warning(( + self.log.warning(( "Current task `{}` doesn't have any loading" " preset for it's context." ).format(current_task_name)) elif not link_context_profiles: - log.warning(( + self.log.warning(( "Current task `{}` doesn't have any" "loading preset for it's linked assets." ).format(current_task_name)) @@ -746,7 +750,7 @@ def build_workfile(self): # Skip if there are no assets. This can happen if only linked mapping # is set and there are no links for his asset. if not assets: - log.warning( + self.log.warning( "Asset does not have linked assets. Nothing to process." ) return @@ -836,7 +840,7 @@ def _filter_build_profiles(self, build_profiles, loaders_by_name): # Check loaders profile_loaders = profile.get("loaders") if not profile_loaders: - log.warning(( + self.log.warning(( "Build profile has missing loaders configuration: {0}" ).format(json.dumps(profile, indent=4))) continue @@ -849,7 +853,7 @@ def _filter_build_profiles(self, build_profiles, loaders_by_name): break if not loaders_match: - log.warning(( + self.log.warning(( "All loaders from Build profile are not available: {0}" ).format(json.dumps(profile, indent=4))) continue @@ -857,7 +861,7 @@ def _filter_build_profiles(self, build_profiles, loaders_by_name): # Check families profile_families = profile.get("families") if not profile_families: - log.warning(( + self.log.warning(( "Build profile is missing families configuration: {0}" ).format(json.dumps(profile, indent=4))) continue @@ -865,7 +869,7 @@ def _filter_build_profiles(self, build_profiles, loaders_by_name): # Check representation names profile_repre_names = profile.get("repre_names") if not profile_repre_names: - log.warning(( + self.log.warning(( "Build profile is missing" " representation names filtering: {0}" ).format(json.dumps(profile, indent=4))) @@ -964,12 +968,12 @@ def load_containers_by_asset_data( build_profiles, loaders_by_name ) if not valid_profiles: - log.warning( + self.log.warning( "There are not valid Workfile profiles. Skipping process." ) return - log.debug("Valid Workfile profiles: {}".format(valid_profiles)) + self.log.debug("Valid Workfile profiles: {}".format(valid_profiles)) subsets_by_id = {} version_by_subset_id = {} @@ -986,7 +990,7 @@ def load_containers_by_asset_data( ) if not subsets_by_id: - log.warning("There are not subsets for asset {0}".format( + self.log.warning("There are not subsets for asset {0}".format( asset_entity["name"] )) return @@ -995,7 +999,7 @@ def load_containers_by_asset_data( subsets_by_id.values(), valid_profiles ) if not profiles_per_subset_id: - log.warning("There are not valid subsets.") + self.log.warning("There are not valid subsets.") return valid_repres_by_subset_id = collections.defaultdict(list) @@ -1022,7 +1026,7 @@ def load_containers_by_asset_data( for repre in repres: msg += "\n## Repre name: `{}`".format(repre["name"]) - log.debug(msg) + self.log.debug(msg) containers = self._load_containers( valid_repres_by_subset_id, subsets_by_id, @@ -1132,13 +1136,13 @@ def _load_containers( except Exception as exc: if exc == pipeline.IncompatibleLoaderError: - log.info(( + self.log.info(( "Loader `{}` is not compatible with" " representation `{}`" ).format(loader_name, repre["name"])) else: - log.error( + self.log.error( "Unexpected error happened during loading", exc_info=True ) @@ -1152,7 +1156,7 @@ def _load_containers( ).format(subset_name) else: msg += " Trying next representation." - log.info(msg) + self.log.info(msg) return loaded_containers From 3edf202815d769e854021fa0efc795c01b6407e1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 11:40:53 +0100 Subject: [PATCH 32/47] moved all avalon context functions to one file --- pype/lib/__init__.py | 28 ++- pype/lib/avalon_context.py | 360 +++++++++++++++++++++++++++++++++++++ pype/lib/lib_old.py | 353 ------------------------------------ 3 files changed, 380 insertions(+), 361 deletions(-) create mode 100644 pype/lib/avalon_context.py diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index a303bf038dc..002689d21cf 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -6,6 +6,17 @@ set_io_database ) +from .avalon_context import ( + is_latest, + any_outdated, + switch_item, + get_asset, + get_hierarchy, + get_subsets, + get_linked_assets, + get_latest_version +) + from .hooks import PypeHook, execute_hook from .applications import ( @@ -20,32 +31,33 @@ _subprocess, get_paths_from_environ, get_ffmpeg_tool_path, - get_hierarchy, add_tool_to_environment, modified_environ, pairwise, grouper, - is_latest, - any_outdated, _rreplace, version_up, - switch_item, _get_host_name, - get_asset, get_version_from_path, get_last_version_from_path, - get_subsets, - get_linked_assets, BuildWorkfile, ffprobe_streams, source_hash, - get_latest_version ) __all__ = [ "get_avalon_database", "set_io_database", + "is_latest", + "any_outdated", + "switch_item", + "get_asset", + "get_hierarchy", + "get_subsets", + "get_linked_assets", + "get_latest_version", + "PypeHook", "execute_hook", diff --git a/pype/lib/avalon_context.py b/pype/lib/avalon_context.py new file mode 100644 index 00000000000..e9cb4c4f5b7 --- /dev/null +++ b/pype/lib/avalon_context.py @@ -0,0 +1,360 @@ +import os +import logging + +from avalon import io +import avalon.api + +log = logging.getLogger("AvalonContext") + + +def is_latest(representation): + """Return whether the representation is from latest version + + Args: + representation (dict): The representation document from the database. + + Returns: + bool: Whether the representation is of latest version. + + """ + + version = io.find_one({"_id": representation['parent']}) + if version["type"] == "master_version": + return True + + # Get highest version under the parent + highest_version = io.find_one({ + "type": "version", + "parent": version["parent"] + }, sort=[("name", -1)], projection={"name": True}) + + if version['name'] == highest_version['name']: + return True + else: + return False + + +def any_outdated(): + """Return whether the current scene has any outdated content""" + + checked = set() + host = avalon.api.registered_host() + for container in host.ls(): + representation = container['representation'] + if representation in checked: + continue + + representation_doc = io.find_one( + { + "_id": io.ObjectId(representation), + "type": "representation" + }, + projection={"parent": True} + ) + if representation_doc and not is_latest(representation_doc): + return True + elif not representation_doc: + log.debug("Container '{objectName}' has an invalid " + "representation, it is missing in the " + "database".format(**container)) + + checked.add(representation) + return False + + +def switch_item(container, + asset_name=None, + subset_name=None, + representation_name=None): + """Switch container asset, subset or representation of a container by name. + + It'll always switch to the latest version - of course a different + approach could be implemented. + + Args: + container (dict): data of the item to switch with + asset_name (str): name of the asset + subset_name (str): name of the subset + representation_name (str): name of the representation + + Returns: + dict + + """ + + if all(not x for x in [asset_name, subset_name, representation_name]): + raise ValueError("Must have at least one change provided to switch.") + + # Collect any of current asset, subset and representation if not provided + # so we can use the original name from those. + if any(not x for x in [asset_name, subset_name, representation_name]): + _id = io.ObjectId(container["representation"]) + representation = io.find_one({"type": "representation", "_id": _id}) + version, subset, asset, project = io.parenthood(representation) + + if asset_name is None: + asset_name = asset["name"] + + if subset_name is None: + subset_name = subset["name"] + + if representation_name is None: + representation_name = representation["name"] + + # Find the new one + asset = io.find_one({ + "name": asset_name, + "type": "asset" + }) + assert asset, ("Could not find asset in the database with the name " + "'%s'" % asset_name) + + subset = io.find_one({ + "name": subset_name, + "type": "subset", + "parent": asset["_id"] + }) + assert subset, ("Could not find subset in the database with the name " + "'%s'" % subset_name) + + version = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + sort=[('name', -1)] + ) + + assert version, "Could not find a version for {}.{}".format( + asset_name, subset_name + ) + + representation = io.find_one({ + "name": representation_name, + "type": "representation", + "parent": version["_id"]} + ) + + assert representation, ("Could not find representation in the database " + "with the name '%s'" % representation_name) + + avalon.api.switch(container, representation) + + return representation + + +def get_asset(asset_name=None): + """ Returning asset document from database """ + if not asset_name: + asset_name = avalon.api.Session["AVALON_ASSET"] + + asset_document = io.find_one({ + "name": asset_name, + "type": "asset" + }) + + if not asset_document: + raise TypeError("Entity \"{}\" was not found in DB".format(asset_name)) + + return asset_document + + +def get_hierarchy(asset_name=None): + """ + Obtain asset hierarchy path string from mongo db + + Returns: + string: asset hierarchy path + + """ + if not asset_name: + asset_name = io.Session.get("AVALON_ASSET", os.environ["AVALON_ASSET"]) + + asset_entity = io.find_one({ + "type": 'asset', + "name": asset_name + }) + + not_set = "PARENTS_NOT_SET" + entity_parents = asset_entity.get("data", {}).get("parents", not_set) + + # If entity already have parents then just return joined + if entity_parents != not_set: + return "/".join(entity_parents) + + # Else query parents through visualParents and store result to entity + hierarchy_items = [] + entity = asset_entity + while True: + parent_id = entity.get("data", {}).get("visualParent") + if not parent_id: + break + entity = io.find_one({"_id": parent_id}) + hierarchy_items.append(entity["name"]) + + # Add parents to entity data for next query + entity_data = asset_entity.get("data", {}) + entity_data["parents"] = hierarchy_items + io.update_many( + {"_id": asset_entity["_id"]}, + {"$set": {"data": entity_data}} + ) + + return "/".join(hierarchy_items) + + +def get_subsets(asset_name, + regex_filter=None, + version=None, + representations=["exr", "dpx"]): + """ + Query subsets with filter on name. + + The method will return all found subsets and its defined version + and subsets. Version could be specified with number. Representation + can be filtered. + + Arguments: + asset_name (str): asset (shot) name + regex_filter (raw): raw string with filter pattern + version (str or int): `last` or number of version + representations (list): list for all representations + + Returns: + dict: subsets with version and representaions in keys + """ + + # query asset from db + asset_io = io.find_one({"type": "asset", "name": asset_name}) + + # check if anything returned + assert asset_io, ( + "Asset not existing. Check correct name: `{}`").format(asset_name) + + # create subsets query filter + filter_query = {"type": "subset", "parent": asset_io["_id"]} + + # add reggex filter string into query filter + if regex_filter: + filter_query.update({"name": {"$regex": r"{}".format(regex_filter)}}) + else: + filter_query.update({"name": {"$regex": r'.*'}}) + + # query all assets + subsets = [s for s in io.find(filter_query)] + + assert subsets, ("No subsets found. Check correct filter. " + "Try this for start `r'.*'`: " + "asset: `{}`").format(asset_name) + + output_dict = {} + # Process subsets + for subset in subsets: + if not version: + version_sel = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + sort=[("name", -1)] + ) + else: + assert isinstance(version, int), "version needs to be `int` type" + version_sel = io.find_one({ + "type": "version", + "parent": subset["_id"], + "name": int(version) + }) + + find_dict = {"type": "representation", + "parent": version_sel["_id"]} + + filter_repr = {"name": {"$in": representations}} + + find_dict.update(filter_repr) + repres_out = [i for i in io.find(find_dict)] + + if len(repres_out) > 0: + output_dict[subset["name"]] = {"version": version_sel, + "representations": repres_out} + + return output_dict + + +def get_linked_assets(asset_entity): + """Return linked assets for `asset_entity`.""" + inputs = asset_entity["data"].get("inputs", []) + inputs = [io.find_one({"_id": x}) for x in inputs] + return inputs + + +def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None): + """Retrieve latest version from `asset_name`, and `subset_name`. + + Do not use if you want to query more than 5 latest versions as this method + query 3 times to mongo for each call. For those cases is better to use + more efficient way, e.g. with help of aggregations. + + Args: + asset_name (str): Name of asset. + subset_name (str): Name of subset. + dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection + with Session. + project_name (str, optional): Find latest version in specific project. + + Returns: + None: If asset, subset or version were not found. + dict: Last version document for entered . + """ + + if not dbcon: + log.debug("Using `avalon.io` for query.") + dbcon = io + # Make sure is installed + io.install() + + if project_name and project_name != dbcon.Session.get("AVALON_PROJECT"): + # `avalon.io` has only `_database` attribute + # but `AvalonMongoDB` has `database` + database = getattr(dbcon, "database", dbcon._database) + collection = database[project_name] + else: + project_name = dbcon.Session.get("AVALON_PROJECT") + collection = dbcon + + log.debug(( + "Getting latest version for Project: \"{}\" Asset: \"{}\"" + " and Subset: \"{}\"" + ).format(project_name, asset_name, subset_name)) + + # Query asset document id by asset name + asset_doc = collection.find_one( + {"type": "asset", "name": asset_name}, + {"_id": True} + ) + if not asset_doc: + log.info( + "Asset \"{}\" was not found in Database.".format(asset_name) + ) + return None + + subset_doc = collection.find_one( + {"type": "subset", "name": subset_name, "parent": asset_doc["_id"]}, + {"_id": True} + ) + if not subset_doc: + log.info( + "Subset \"{}\" was not found in Database.".format(subset_name) + ) + return None + + version_doc = collection.find_one( + {"type": "version", "parent": subset_doc["_id"]}, + sort=[("name", -1)], + ) + if not version_doc: + log.info( + "Subset \"{}\" does not have any version yet.".format(subset_name) + ) + return None + return version_doc diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 1f608e368e3..c559324a5e7 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -135,50 +135,6 @@ def _subprocess(*args, **kwargs): return full_output -def get_hierarchy(asset_name=None): - """ - Obtain asset hierarchy path string from mongo db - - Returns: - string: asset hierarchy path - - """ - if not asset_name: - asset_name = io.Session.get("AVALON_ASSET", os.environ["AVALON_ASSET"]) - - asset_entity = io.find_one({ - "type": 'asset', - "name": asset_name - }) - - not_set = "PARENTS_NOT_SET" - entity_parents = asset_entity.get("data", {}).get("parents", not_set) - - # If entity already have parents then just return joined - if entity_parents != not_set: - return "/".join(entity_parents) - - # Else query parents through visualParents and store result to entity - hierarchy_items = [] - entity = asset_entity - while True: - parent_id = entity.get("data", {}).get("visualParent") - if not parent_id: - break - entity = io.find_one({"_id": parent_id}) - hierarchy_items.append(entity["name"]) - - # Add parents to entity data for next query - entity_data = asset_entity.get("data", {}) - entity_data["parents"] = hierarchy_items - io.update_many( - {"_id": asset_entity["_id"]}, - {"$set": {"data": entity_data}} - ) - - return "/".join(hierarchy_items) - - def add_tool_to_environment(tools): """ It is adding dynamic environment to os environment. @@ -247,61 +203,6 @@ def grouper(iterable, n, fillvalue=None): return itertools.izip_longest(fillvalue=fillvalue, *args) -def is_latest(representation): - """Return whether the representation is from latest version - - Args: - representation (dict): The representation document from the database. - - Returns: - bool: Whether the representation is of latest version. - - """ - - version = io.find_one({"_id": representation['parent']}) - if version["type"] == "master_version": - return True - - # Get highest version under the parent - highest_version = io.find_one({ - "type": "version", - "parent": version["parent"] - }, sort=[("name", -1)], projection={"name": True}) - - if version['name'] == highest_version['name']: - return True - else: - return False - - -def any_outdated(): - """Return whether the current scene has any outdated content""" - - checked = set() - host = avalon.api.registered_host() - for container in host.ls(): - representation = container['representation'] - if representation in checked: - continue - - representation_doc = io.find_one( - { - "_id": io.ObjectId(representation), - "type": "representation" - }, - projection={"parent": True} - ) - if representation_doc and not is_latest(representation_doc): - return True - elif not representation_doc: - log.debug("Container '{objectName}' has an invalid " - "representation, it is missing in the " - "database".format(**container)) - - checked.add(representation) - return False - - def _rreplace(s, a, b, n=1): """Replace a with b in string s from right side n times""" return b.join(s.rsplit(a, n)) @@ -360,87 +261,6 @@ def version_up(filepath): return new_filename -def switch_item(container, - asset_name=None, - subset_name=None, - representation_name=None): - """Switch container asset, subset or representation of a container by name. - - It'll always switch to the latest version - of course a different - approach could be implemented. - - Args: - container (dict): data of the item to switch with - asset_name (str): name of the asset - subset_name (str): name of the subset - representation_name (str): name of the representation - - Returns: - dict - - """ - - if all(not x for x in [asset_name, subset_name, representation_name]): - raise ValueError("Must have at least one change provided to switch.") - - # Collect any of current asset, subset and representation if not provided - # so we can use the original name from those. - if any(not x for x in [asset_name, subset_name, representation_name]): - _id = io.ObjectId(container["representation"]) - representation = io.find_one({"type": "representation", "_id": _id}) - version, subset, asset, project = io.parenthood(representation) - - if asset_name is None: - asset_name = asset["name"] - - if subset_name is None: - subset_name = subset["name"] - - if representation_name is None: - representation_name = representation["name"] - - # Find the new one - asset = io.find_one({ - "name": asset_name, - "type": "asset" - }) - assert asset, ("Could not find asset in the database with the name " - "'%s'" % asset_name) - - subset = io.find_one({ - "name": subset_name, - "type": "subset", - "parent": asset["_id"] - }) - assert subset, ("Could not find subset in the database with the name " - "'%s'" % subset_name) - - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[('name', -1)] - ) - - assert version, "Could not find a version for {}.{}".format( - asset_name, subset_name - ) - - representation = io.find_one({ - "name": representation_name, - "type": "representation", - "parent": version["_id"]} - ) - - assert representation, ("Could not find representation in the database " - "with the name '%s'" % representation_name) - - avalon.api.switch(container, representation) - - return representation - - def _get_host_name(): _host = avalon.api.registered_host() @@ -448,22 +268,6 @@ def _get_host_name(): return _host.__name__.rsplit(".", 1)[-1] -def get_asset(asset_name=None): - """ Returning asset document from database """ - if not asset_name: - asset_name = avalon.api.Session["AVALON_ASSET"] - - asset_document = io.find_one({ - "name": asset_name, - "type": "asset" - }) - - if not asset_document: - raise TypeError("Entity \"{}\" was not found in DB".format(asset_name)) - - return asset_document - - def get_version_from_path(file): """ Finds version number in file path string @@ -523,91 +327,6 @@ def get_last_version_from_path(path_dir, filter): return None -def get_subsets(asset_name, - regex_filter=None, - version=None, - representations=["exr", "dpx"]): - """ - Query subsets with filter on name. - - The method will return all found subsets and its defined version - and subsets. Version could be specified with number. Representation - can be filtered. - - Arguments: - asset_name (str): asset (shot) name - regex_filter (raw): raw string with filter pattern - version (str or int): `last` or number of version - representations (list): list for all representations - - Returns: - dict: subsets with version and representaions in keys - """ - - # query asset from db - asset_io = io.find_one({"type": "asset", "name": asset_name}) - - # check if anything returned - assert asset_io, ( - "Asset not existing. Check correct name: `{}`").format(asset_name) - - # create subsets query filter - filter_query = {"type": "subset", "parent": asset_io["_id"]} - - # add reggex filter string into query filter - if regex_filter: - filter_query.update({"name": {"$regex": r"{}".format(regex_filter)}}) - else: - filter_query.update({"name": {"$regex": r'.*'}}) - - # query all assets - subsets = [s for s in io.find(filter_query)] - - assert subsets, ("No subsets found. Check correct filter. " - "Try this for start `r'.*'`: " - "asset: `{}`").format(asset_name) - - output_dict = {} - # Process subsets - for subset in subsets: - if not version: - version_sel = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - else: - assert isinstance(version, int), "version needs to be `int` type" - version_sel = io.find_one({ - "type": "version", - "parent": subset["_id"], - "name": int(version) - }) - - find_dict = {"type": "representation", - "parent": version_sel["_id"]} - - filter_repr = {"name": {"$in": representations}} - - find_dict.update(filter_repr) - repres_out = [i for i in io.find(find_dict)] - - if len(repres_out) > 0: - output_dict[subset["name"]] = {"version": version_sel, - "representations": repres_out} - - return output_dict - - -def get_linked_assets(asset_entity): - """Return linked assets for `asset_entity`.""" - inputs = asset_entity["data"].get("inputs", []) - inputs = [io.find_one({"_id": x}) for x in inputs] - return inputs - - class BuildWorkfile: """Wrapper for build workfile process. @@ -1307,75 +1026,3 @@ def source_hash(filepath, *args): time = str(os.path.getmtime(filepath)) size = str(os.path.getsize(filepath)) return "|".join([file_name, time, size] + list(args)).replace(".", ",") - - -def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None): - """Retrieve latest version from `asset_name`, and `subset_name`. - - Do not use if you want to query more than 5 latest versions as this method - query 3 times to mongo for each call. For those cases is better to use - more efficient way, e.g. with help of aggregations. - - Args: - asset_name (str): Name of asset. - subset_name (str): Name of subset. - dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection - with Session. - project_name (str, optional): Find latest version in specific project. - - Returns: - None: If asset, subset or version were not found. - dict: Last version document for entered . - """ - - if not dbcon: - log.debug("Using `avalon.io` for query.") - dbcon = io - # Make sure is installed - io.install() - - if project_name and project_name != dbcon.Session.get("AVALON_PROJECT"): - # `avalon.io` has only `_database` attribute - # but `AvalonMongoDB` has `database` - database = getattr(dbcon, "database", dbcon._database) - collection = database[project_name] - else: - project_name = dbcon.Session.get("AVALON_PROJECT") - collection = dbcon - - log.debug(( - "Getting latest version for Project: \"{}\" Asset: \"{}\"" - " and Subset: \"{}\"" - ).format(project_name, asset_name, subset_name)) - - # Query asset document id by asset name - asset_doc = collection.find_one( - {"type": "asset", "name": asset_name}, - {"_id": True} - ) - if not asset_doc: - log.info( - "Asset \"{}\" was not found in Database.".format(asset_name) - ) - return None - - subset_doc = collection.find_one( - {"type": "subset", "name": subset_name, "parent": asset_doc["_id"]}, - {"_id": True} - ) - if not subset_doc: - log.info( - "Subset \"{}\" was not found in Database.".format(subset_name) - ) - return None - - version_doc = collection.find_one( - {"type": "version", "parent": subset_doc["_id"]}, - sort=[("name", -1)], - ) - if not version_doc: - log.info( - "Subset \"{}\" does not have any version yet.".format(subset_name) - ) - return None - return version_doc From c116e8042b6ca54891e2db5912073ccb008d13a6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 11:42:47 +0100 Subject: [PATCH 33/47] moved `get_subsets` to collect audio in celaction collector which is only place where is used --- pype/lib/__init__.py | 2 - pype/lib/avalon_context.py | 78 ----------------- .../celaction/publish/collect_audio.py | 84 ++++++++++++++++++- 3 files changed, 82 insertions(+), 82 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 002689d21cf..59411db9de8 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -12,7 +12,6 @@ switch_item, get_asset, get_hierarchy, - get_subsets, get_linked_assets, get_latest_version ) @@ -54,7 +53,6 @@ "switch_item", "get_asset", "get_hierarchy", - "get_subsets", "get_linked_assets", "get_latest_version", diff --git a/pype/lib/avalon_context.py b/pype/lib/avalon_context.py index e9cb4c4f5b7..099b48967b3 100644 --- a/pype/lib/avalon_context.py +++ b/pype/lib/avalon_context.py @@ -203,84 +203,6 @@ def get_hierarchy(asset_name=None): return "/".join(hierarchy_items) -def get_subsets(asset_name, - regex_filter=None, - version=None, - representations=["exr", "dpx"]): - """ - Query subsets with filter on name. - - The method will return all found subsets and its defined version - and subsets. Version could be specified with number. Representation - can be filtered. - - Arguments: - asset_name (str): asset (shot) name - regex_filter (raw): raw string with filter pattern - version (str or int): `last` or number of version - representations (list): list for all representations - - Returns: - dict: subsets with version and representaions in keys - """ - - # query asset from db - asset_io = io.find_one({"type": "asset", "name": asset_name}) - - # check if anything returned - assert asset_io, ( - "Asset not existing. Check correct name: `{}`").format(asset_name) - - # create subsets query filter - filter_query = {"type": "subset", "parent": asset_io["_id"]} - - # add reggex filter string into query filter - if regex_filter: - filter_query.update({"name": {"$regex": r"{}".format(regex_filter)}}) - else: - filter_query.update({"name": {"$regex": r'.*'}}) - - # query all assets - subsets = [s for s in io.find(filter_query)] - - assert subsets, ("No subsets found. Check correct filter. " - "Try this for start `r'.*'`: " - "asset: `{}`").format(asset_name) - - output_dict = {} - # Process subsets - for subset in subsets: - if not version: - version_sel = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - else: - assert isinstance(version, int), "version needs to be `int` type" - version_sel = io.find_one({ - "type": "version", - "parent": subset["_id"], - "name": int(version) - }) - - find_dict = {"type": "representation", - "parent": version_sel["_id"]} - - filter_repr = {"name": {"$in": representations}} - - find_dict.update(filter_repr) - repres_out = [i for i in io.find(find_dict)] - - if len(repres_out) > 0: - output_dict[subset["name"]] = {"version": version_sel, - "representations": repres_out} - - return output_dict - - def get_linked_assets(asset_entity): """Return linked assets for `asset_entity`.""" inputs = asset_entity["data"].get("inputs", []) diff --git a/pype/plugins/celaction/publish/collect_audio.py b/pype/plugins/celaction/publish/collect_audio.py index c29e212d808..c92e4fd868b 100644 --- a/pype/plugins/celaction/publish/collect_audio.py +++ b/pype/plugins/celaction/publish/collect_audio.py @@ -1,6 +1,8 @@ -import pyblish.api import os +import pyblish.api +from avalon import io + import pype.api as pype from pprint import pformat @@ -15,7 +17,7 @@ def process(self, context): asset_entity = context.data["assetEntity"] # get all available representations - subsets = pype.get_subsets(asset_entity["name"], + subsets = self.get_subsets(asset_entity["name"], representations=["audio", "wav"] ) self.log.info(f"subsets is: {pformat(subsets)}") @@ -39,3 +41,81 @@ def process(self, context): 'audio_file: {}, has been added to context'.format(audio_file)) else: self.log.warning("Couldn't find any audio file on Ftrack.") + + def get_subsets( + self, + asset_name, + regex_filter=None, + version=None, + representations=["exr", "dpx"] + ): + """ + Query subsets with filter on name. + + The method will return all found subsets and its defined version + and subsets. Version could be specified with number. Representation + can be filtered. + + Arguments: + asset_name (str): asset (shot) name + regex_filter (raw): raw string with filter pattern + version (str or int): `last` or number of version + representations (list): list for all representations + + Returns: + dict: subsets with version and representaions in keys + """ + + # query asset from db + asset_io = io.find_one({"type": "asset", "name": asset_name}) + + # check if anything returned + assert asset_io, ( + "Asset not existing. Check correct name: `{}`").format(asset_name) + + # create subsets query filter + filter_query = {"type": "subset", "parent": asset_io["_id"]} + + # add reggex filter string into query filter + if regex_filter: + filter_query["name"] = {"$regex": r"{}".format(regex_filter)} + + # query all assets + subsets = list(io.find(filter_query)) + + assert subsets, ("No subsets found. Check correct filter. " + "Try this for start `r'.*'`: " + "asset: `{}`").format(asset_name) + + output_dict = {} + # Process subsets + for subset in subsets: + if not version: + version_sel = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + sort=[("name", -1)] + ) + else: + assert isinstance(version, int), "version needs to be `int` type" + version_sel = io.find_one({ + "type": "version", + "parent": subset["_id"], + "name": int(version) + }) + + find_dict = {"type": "representation", + "parent": version_sel["_id"]} + + filter_repr = {"name": {"$in": representations}} + + find_dict.update(filter_repr) + repres_out = [i for i in io.find(find_dict)] + + if len(repres_out) > 0: + output_dict[subset["name"]] = {"version": version_sel, + "representations": repres_out} + + return output_dict From 36cd349c6a5b2c38cbd58792d9758b1bcab00241 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 11:51:06 +0100 Subject: [PATCH 34/47] switch_item moved to fusion.lib as it's only place where is used --- pype/hosts/fusion/lib.py | 83 ++++++++++++++++++- .../fusion/scripts/fusion_switch_shot.py | 2 +- pype/lib/__init__.py | 2 - pype/lib/avalon_context.py | 81 ------------------ pype/scripts/fusion_switch_shot.py | 2 +- 5 files changed, 84 insertions(+), 86 deletions(-) diff --git a/pype/hosts/fusion/lib.py b/pype/hosts/fusion/lib.py index f2846c966a2..77866fde9d7 100644 --- a/pype/hosts/fusion/lib.py +++ b/pype/hosts/fusion/lib.py @@ -2,7 +2,7 @@ from avalon.vendor.Qt import QtGui import avalon.fusion - +from avalon import io self = sys.modules[__name__] self._project = None @@ -59,3 +59,84 @@ def get_additional_data(container): return {"color": QtGui.QColor.fromRgbF(tile_color["R"], tile_color["G"], tile_color["B"])} + + +def switch_item(container, + asset_name=None, + subset_name=None, + representation_name=None): + """Switch container asset, subset or representation of a container by name. + + It'll always switch to the latest version - of course a different + approach could be implemented. + + Args: + container (dict): data of the item to switch with + asset_name (str): name of the asset + subset_name (str): name of the subset + representation_name (str): name of the representation + + Returns: + dict + + """ + + if all(not x for x in [asset_name, subset_name, representation_name]): + raise ValueError("Must have at least one change provided to switch.") + + # Collect any of current asset, subset and representation if not provided + # so we can use the original name from those. + if any(not x for x in [asset_name, subset_name, representation_name]): + _id = io.ObjectId(container["representation"]) + representation = io.find_one({"type": "representation", "_id": _id}) + version, subset, asset, project = io.parenthood(representation) + + if asset_name is None: + asset_name = asset["name"] + + if subset_name is None: + subset_name = subset["name"] + + if representation_name is None: + representation_name = representation["name"] + + # Find the new one + asset = io.find_one({ + "name": asset_name, + "type": "asset" + }) + assert asset, ("Could not find asset in the database with the name " + "'%s'" % asset_name) + + subset = io.find_one({ + "name": subset_name, + "type": "subset", + "parent": asset["_id"] + }) + assert subset, ("Could not find subset in the database with the name " + "'%s'" % subset_name) + + version = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + sort=[('name', -1)] + ) + + assert version, "Could not find a version for {}.{}".format( + asset_name, subset_name + ) + + representation = io.find_one({ + "name": representation_name, + "type": "representation", + "parent": version["_id"]} + ) + + assert representation, ("Could not find representation in the database " + "with the name '%s'" % representation_name) + + avalon.api.switch(container, representation) + + return representation diff --git a/pype/hosts/fusion/scripts/fusion_switch_shot.py b/pype/hosts/fusion/scripts/fusion_switch_shot.py index a3f2116db81..ed657cb6127 100644 --- a/pype/hosts/fusion/scripts/fusion_switch_shot.py +++ b/pype/hosts/fusion/scripts/fusion_switch_shot.py @@ -234,7 +234,7 @@ def switch(asset_name, filepath=None, new=True): representations = [] for container in containers: try: - representation = pype.switch_item( + representation = fusion_lib.switch_item( container, asset_name=asset_name) representations.append(representation) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 59411db9de8..a93d371b2b2 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -9,7 +9,6 @@ from .avalon_context import ( is_latest, any_outdated, - switch_item, get_asset, get_hierarchy, get_linked_assets, @@ -50,7 +49,6 @@ "is_latest", "any_outdated", - "switch_item", "get_asset", "get_hierarchy", "get_linked_assets", diff --git a/pype/lib/avalon_context.py b/pype/lib/avalon_context.py index 099b48967b3..813d244d72e 100644 --- a/pype/lib/avalon_context.py +++ b/pype/lib/avalon_context.py @@ -62,87 +62,6 @@ def any_outdated(): return False -def switch_item(container, - asset_name=None, - subset_name=None, - representation_name=None): - """Switch container asset, subset or representation of a container by name. - - It'll always switch to the latest version - of course a different - approach could be implemented. - - Args: - container (dict): data of the item to switch with - asset_name (str): name of the asset - subset_name (str): name of the subset - representation_name (str): name of the representation - - Returns: - dict - - """ - - if all(not x for x in [asset_name, subset_name, representation_name]): - raise ValueError("Must have at least one change provided to switch.") - - # Collect any of current asset, subset and representation if not provided - # so we can use the original name from those. - if any(not x for x in [asset_name, subset_name, representation_name]): - _id = io.ObjectId(container["representation"]) - representation = io.find_one({"type": "representation", "_id": _id}) - version, subset, asset, project = io.parenthood(representation) - - if asset_name is None: - asset_name = asset["name"] - - if subset_name is None: - subset_name = subset["name"] - - if representation_name is None: - representation_name = representation["name"] - - # Find the new one - asset = io.find_one({ - "name": asset_name, - "type": "asset" - }) - assert asset, ("Could not find asset in the database with the name " - "'%s'" % asset_name) - - subset = io.find_one({ - "name": subset_name, - "type": "subset", - "parent": asset["_id"] - }) - assert subset, ("Could not find subset in the database with the name " - "'%s'" % subset_name) - - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[('name', -1)] - ) - - assert version, "Could not find a version for {}.{}".format( - asset_name, subset_name - ) - - representation = io.find_one({ - "name": representation_name, - "type": "representation", - "parent": version["_id"]} - ) - - assert representation, ("Could not find representation in the database " - "with the name '%s'" % representation_name) - - avalon.api.switch(container, representation) - - return representation - - def get_asset(asset_name=None): """ Returning asset document from database """ if not asset_name: diff --git a/pype/scripts/fusion_switch_shot.py b/pype/scripts/fusion_switch_shot.py index f936b7d8e0a..5791220acda 100644 --- a/pype/scripts/fusion_switch_shot.py +++ b/pype/scripts/fusion_switch_shot.py @@ -191,7 +191,7 @@ def switch(asset_name, filepath=None, new=True): representations = [] for container in containers: try: - representation = pype.switch_item(container, + representation = fusion_lib.switch_item(container, asset_name=asset_name) representations.append(representation) except Exception as e: From 833d6ffdff03a308451dabc6452931bcf05bdee3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 11:54:30 +0100 Subject: [PATCH 35/47] moved BuildWorkfile to avalon context --- pype/lib/__init__.py | 4 +- pype/lib/avalon_context.py | 653 ++++++++++++++++++++++++++++++++++++- pype/lib/lib_old.py | 647 ------------------------------------ 3 files changed, 655 insertions(+), 649 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index a93d371b2b2..f807fe894a5 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -12,7 +12,8 @@ get_asset, get_hierarchy, get_linked_assets, - get_latest_version + get_latest_version, + BuildWorkfile ) from .hooks import PypeHook, execute_hook @@ -53,6 +54,7 @@ "get_hierarchy", "get_linked_assets", "get_latest_version", + "BuildWorkfile", "PypeHook", "execute_hook", diff --git a/pype/lib/avalon_context.py b/pype/lib/avalon_context.py index 813d244d72e..56abc4aee6f 100644 --- a/pype/lib/avalon_context.py +++ b/pype/lib/avalon_context.py @@ -1,7 +1,11 @@ import os +import json +import re import logging +import collections -from avalon import io +from avalon import io, pipeline +from ..api import config import avalon.api log = logging.getLogger("AvalonContext") @@ -199,3 +203,650 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None): ) return None return version_doc + + +class BuildWorkfile: + """Wrapper for build workfile process. + + Load representations for current context by build presets. Build presets + are host related, since each host has it's loaders. + """ + + log = logging.getLogger("BuildWorkfile") + + @staticmethod + def map_subsets_by_family(subsets): + subsets_by_family = collections.defaultdict(list) + for subset in subsets: + family = subset["data"].get("family") + if not family: + families = subset["data"].get("families") + if not families: + continue + family = families[0] + + subsets_by_family[family].append(subset) + return subsets_by_family + + def process(self): + """Main method of this wrapper. + + Building of workfile is triggered and is possible to implement + post processing of loaded containers if necessary. + """ + containers = self.build_workfile() + + return containers + + def build_workfile(self): + """Prepares and load containers into workfile. + + Loads latest versions of current and linked assets to workfile by logic + stored in Workfile profiles from presets. Profiles are set by host, + filtered by current task name and used by families. + + Each family can specify representation names and loaders for + representations and first available and successful loaded + representation is returned as container. + + At the end you'll get list of loaded containers per each asset. + + loaded_containers [{ + "asset_entity": , + "containers": [, , ...] + }, { + "asset_entity": , + "containers": [, ...] + }, { + ... + }] + """ + # Get current asset name and entity + current_asset_name = io.Session["AVALON_ASSET"] + current_asset_entity = io.find_one({ + "type": "asset", + "name": current_asset_name + }) + + # Skip if asset was not found + if not current_asset_entity: + print("Asset entity with name `{}` was not found".format( + current_asset_name + )) + return + + # Prepare available loaders + loaders_by_name = {} + for loader in avalon.api.discover(avalon.api.Loader): + loader_name = loader.__name__ + if loader_name in loaders_by_name: + raise KeyError( + "Duplicated loader name {0}!".format(loader_name) + ) + loaders_by_name[loader_name] = loader + + # Skip if there are any loaders + if not loaders_by_name: + self.log.warning("There are no registered loaders.") + return + + # Get current task name + current_task_name = io.Session["AVALON_TASK"] + + # Load workfile presets for task + self.build_presets = self.get_build_presets(current_task_name) + + # Skip if there are any presets for task + if not self.build_presets: + self.log.warning( + "Current task `{}` does not have any loading preset.".format( + current_task_name + ) + ) + return + + # Get presets for loading current asset + current_context_profiles = self.build_presets.get("current_context") + # Get presets for loading linked assets + link_context_profiles = self.build_presets.get("linked_assets") + # Skip if both are missing + if not current_context_profiles and not link_context_profiles: + self.log.warning( + "Current task `{}` has empty loading preset.".format( + current_task_name + ) + ) + return + + elif not current_context_profiles: + self.log.warning(( + "Current task `{}` doesn't have any loading" + " preset for it's context." + ).format(current_task_name)) + + elif not link_context_profiles: + self.log.warning(( + "Current task `{}` doesn't have any" + "loading preset for it's linked assets." + ).format(current_task_name)) + + # Prepare assets to process by workfile presets + assets = [] + current_asset_id = None + if current_context_profiles: + # Add current asset entity if preset has current context set + assets.append(current_asset_entity) + current_asset_id = current_asset_entity["_id"] + + if link_context_profiles: + # Find and append linked assets if preset has set linked mapping + link_assets = get_linked_assets(current_asset_entity) + if link_assets: + assets.extend(link_assets) + + # Skip if there are no assets. This can happen if only linked mapping + # is set and there are no links for his asset. + if not assets: + self.log.warning( + "Asset does not have linked assets. Nothing to process." + ) + return + + # Prepare entities from database for assets + prepared_entities = self._collect_last_version_repres(assets) + + # Load containers by prepared entities and presets + loaded_containers = [] + # - Current asset containers + if current_asset_id and current_asset_id in prepared_entities: + current_context_data = prepared_entities.pop(current_asset_id) + loaded_data = self.load_containers_by_asset_data( + current_context_data, current_context_profiles, loaders_by_name + ) + if loaded_data: + loaded_containers.append(loaded_data) + + # - Linked assets container + for linked_asset_data in prepared_entities.values(): + loaded_data = self.load_containers_by_asset_data( + linked_asset_data, link_context_profiles, loaders_by_name + ) + if loaded_data: + loaded_containers.append(loaded_data) + + # Return list of loaded containers + return loaded_containers + + def get_build_presets(self, task_name): + """ Returns presets to build workfile for task name. + + Presets are loaded for current project set in + io.Session["AVALON_PROJECT"], filtered by registered host + and entered task name. + + :param task_name: Task name used for filtering build presets. + :type task_name: str + :return: preset per eneter task + :rtype: dict | None + """ + host_name = avalon.api.registered_host().__name__.rsplit(".", 1)[-1] + presets = config.get_presets(io.Session["AVALON_PROJECT"]) + # Get presets for host + build_presets = ( + presets["plugins"] + .get(host_name, {}) + .get("workfile_build") + ) + if not build_presets: + return + + task_name_low = task_name.lower() + per_task_preset = None + for preset in build_presets: + preset_tasks = preset.get("tasks") or [] + preset_tasks_low = [task.lower() for task in preset_tasks] + if task_name_low in preset_tasks_low: + per_task_preset = preset + break + + return per_task_preset + + def _filter_build_profiles(self, build_profiles, loaders_by_name): + """ Filter build profiles by loaders and prepare process data. + + Valid profile must have "loaders", "families" and "repre_names" keys + with valid values. + - "loaders" expects list of strings representing possible loaders. + - "families" expects list of strings for filtering + by main subset family. + - "repre_names" expects list of strings for filtering by + representation name. + + Lowered "families" and "repre_names" are prepared for each profile with + all required keys. + + :param build_profiles: Profiles for building workfile. + :type build_profiles: dict + :param loaders_by_name: Available loaders per name. + :type loaders_by_name: dict + :return: Filtered and prepared profiles. + :rtype: list + """ + valid_profiles = [] + for profile in build_profiles: + # Check loaders + profile_loaders = profile.get("loaders") + if not profile_loaders: + self.log.warning(( + "Build profile has missing loaders configuration: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Check if any loader is available + loaders_match = False + for loader_name in profile_loaders: + if loader_name in loaders_by_name: + loaders_match = True + break + + if not loaders_match: + self.log.warning(( + "All loaders from Build profile are not available: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Check families + profile_families = profile.get("families") + if not profile_families: + self.log.warning(( + "Build profile is missing families configuration: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Check representation names + profile_repre_names = profile.get("repre_names") + if not profile_repre_names: + self.log.warning(( + "Build profile is missing" + " representation names filtering: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Prepare lowered families and representation names + profile["families_lowered"] = [ + fam.lower() for fam in profile_families + ] + profile["repre_names_lowered"] = [ + name.lower() for name in profile_repre_names + ] + + valid_profiles.append(profile) + + return valid_profiles + + def _prepare_profile_for_subsets(self, subsets, profiles): + """Select profile for each subset byt it's data. + + Profiles are filtered for each subset individually. + Profile is filtered by subset's family, optionally by name regex and + representation names set in profile. + It is possible to not find matching profile for subset, in that case + subset is skipped and it is possible that none of subsets have + matching profile. + + :param subsets: Subset documents. + :type subsets: list + :param profiles: Build profiles. + :type profiles: dict + :return: Profile by subset's id. + :rtype: dict + """ + # Prepare subsets + subsets_by_family = self.map_subsets_by_family(subsets) + + profiles_per_subset_id = {} + for family, subsets in subsets_by_family.items(): + family_low = family.lower() + for profile in profiles: + # Skip profile if does not contain family + if family_low not in profile["families_lowered"]: + continue + + # Precompile name filters as regexes + profile_regexes = profile.get("subset_name_filters") + if profile_regexes: + _profile_regexes = [] + for regex in profile_regexes: + _profile_regexes.append(re.compile(regex)) + profile_regexes = _profile_regexes + + # TODO prepare regex compilation + for subset in subsets: + # Verify regex filtering (optional) + if profile_regexes: + valid = False + for pattern in profile_regexes: + if re.match(pattern, subset["name"]): + valid = True + break + + if not valid: + continue + + profiles_per_subset_id[subset["_id"]] = profile + + # break profiles loop on finding the first matching profile + break + return profiles_per_subset_id + + def load_containers_by_asset_data( + self, asset_entity_data, build_profiles, loaders_by_name + ): + """Load containers for entered asset entity by Build profiles. + + :param asset_entity_data: Prepared data with subsets, last version + and representations for specific asset. + :type asset_entity_data: dict + :param build_profiles: Build profiles. + :type build_profiles: dict + :param loaders_by_name: Available loaders per name. + :type loaders_by_name: dict + :return: Output contains asset document and loaded containers. + :rtype: dict + """ + + # Make sure all data are not empty + if not asset_entity_data or not build_profiles or not loaders_by_name: + return + + asset_entity = asset_entity_data["asset_entity"] + + valid_profiles = self._filter_build_profiles( + build_profiles, loaders_by_name + ) + if not valid_profiles: + self.log.warning( + "There are not valid Workfile profiles. Skipping process." + ) + return + + self.log.debug("Valid Workfile profiles: {}".format(valid_profiles)) + + subsets_by_id = {} + version_by_subset_id = {} + repres_by_version_id = {} + for subset_id, in_data in asset_entity_data["subsets"].items(): + subset_entity = in_data["subset_entity"] + subsets_by_id[subset_entity["_id"]] = subset_entity + + version_data = in_data["version"] + version_entity = version_data["version_entity"] + version_by_subset_id[subset_id] = version_entity + repres_by_version_id[version_entity["_id"]] = ( + version_data["repres"] + ) + + if not subsets_by_id: + self.log.warning("There are not subsets for asset {0}".format( + asset_entity["name"] + )) + return + + profiles_per_subset_id = self._prepare_profile_for_subsets( + subsets_by_id.values(), valid_profiles + ) + if not profiles_per_subset_id: + self.log.warning("There are not valid subsets.") + return + + valid_repres_by_subset_id = collections.defaultdict(list) + for subset_id, profile in profiles_per_subset_id.items(): + profile_repre_names = profile["repre_names_lowered"] + + version_entity = version_by_subset_id[subset_id] + version_id = version_entity["_id"] + repres = repres_by_version_id[version_id] + for repre in repres: + repre_name_low = repre["name"].lower() + if repre_name_low in profile_repre_names: + valid_repres_by_subset_id[subset_id].append(repre) + + # DEBUG message + msg = "Valid representations for Asset: `{}`".format( + asset_entity["name"] + ) + for subset_id, repres in valid_repres_by_subset_id.items(): + subset = subsets_by_id[subset_id] + msg += "\n# Subset Name/ID: `{}`/{}".format( + subset["name"], subset_id + ) + for repre in repres: + msg += "\n## Repre name: `{}`".format(repre["name"]) + + self.log.debug(msg) + + containers = self._load_containers( + valid_repres_by_subset_id, subsets_by_id, + profiles_per_subset_id, loaders_by_name + ) + + return { + "asset_entity": asset_entity, + "containers": containers + } + + def _load_containers( + self, repres_by_subset_id, subsets_by_id, + profiles_per_subset_id, loaders_by_name + ): + """Real load by collected data happens here. + + Loading of representations per subset happens here. Each subset can + loads one representation. Loading is tried in specific order. + Representations are tried to load by names defined in configuration. + If subset has representation matching representation name each loader + is tried to load it until any is successful. If none of them was + successful then next reprensentation name is tried. + Subset process loop ends when any representation is loaded or + all matching representations were already tried. + + :param repres_by_subset_id: Available representations mapped + by their parent (subset) id. + :type repres_by_subset_id: dict + :param subsets_by_id: Subset documents mapped by their id. + :type subsets_by_id: dict + :param profiles_per_subset_id: Build profiles mapped by subset id. + :type profiles_per_subset_id: dict + :param loaders_by_name: Available loaders per name. + :type loaders_by_name: dict + :return: Objects of loaded containers. + :rtype: list + """ + loaded_containers = [] + + # Get subset id order from build presets. + build_presets = self.build_presets.get("current_context", []) + build_presets += self.build_presets.get("linked_assets", []) + subset_ids_ordered = [] + for preset in build_presets: + for preset_family in preset["families"]: + for id, subset in subsets_by_id.items(): + if preset_family not in subset["data"].get("families", []): + continue + + subset_ids_ordered.append(id) + + # Order representations from subsets. + print("repres_by_subset_id", repres_by_subset_id) + representations_ordered = [] + representations = [] + for id in subset_ids_ordered: + for subset_id, repres in repres_by_subset_id.items(): + if repres in representations: + continue + + if id == subset_id: + representations_ordered.append((subset_id, repres)) + representations.append(repres) + + print("representations", representations) + + # Load ordered reprensentations. + for subset_id, repres in representations_ordered: + subset_name = subsets_by_id[subset_id]["name"] + + profile = profiles_per_subset_id[subset_id] + loaders_last_idx = len(profile["loaders"]) - 1 + repre_names_last_idx = len(profile["repre_names_lowered"]) - 1 + + repre_by_low_name = { + repre["name"].lower(): repre for repre in repres + } + + is_loaded = False + for repre_name_idx, profile_repre_name in enumerate( + profile["repre_names_lowered"] + ): + # Break iteration if representation was already loaded + if is_loaded: + break + + repre = repre_by_low_name.get(profile_repre_name) + if not repre: + continue + + for loader_idx, loader_name in enumerate(profile["loaders"]): + if is_loaded: + break + + loader = loaders_by_name.get(loader_name) + if not loader: + continue + try: + container = avalon.api.load( + loader, + repre["_id"], + name=subset_name + ) + loaded_containers.append(container) + is_loaded = True + + except Exception as exc: + if exc == pipeline.IncompatibleLoaderError: + self.log.info(( + "Loader `{}` is not compatible with" + " representation `{}`" + ).format(loader_name, repre["name"])) + + else: + self.log.error( + "Unexpected error happened during loading", + exc_info=True + ) + + msg = "Loading failed." + if loader_idx < loaders_last_idx: + msg += " Trying next loader." + elif repre_name_idx < repre_names_last_idx: + msg += ( + " Loading of subset `{}` was not successful." + ).format(subset_name) + else: + msg += " Trying next representation." + self.log.info(msg) + + return loaded_containers + + def _collect_last_version_repres(self, asset_entities): + """Collect subsets, versions and representations for asset_entities. + + :param asset_entities: Asset entities for which want to find data + :type asset_entities: list + :return: collected entities + :rtype: dict + + Example output: + ``` + { + {Asset ID}: { + "asset_entity": , + "subsets": { + {Subset ID}: { + "subset_entity": , + "version": { + "version_entity": , + "repres": [ + , , ... + ] + } + }, + ... + } + }, + ... + } + output[asset_id]["subsets"][subset_id]["version"]["repres"] + ``` + """ + + if not asset_entities: + return {} + + asset_entity_by_ids = {asset["_id"]: asset for asset in asset_entities} + + subsets = list(io.find({ + "type": "subset", + "parent": {"$in": asset_entity_by_ids.keys()} + })) + subset_entity_by_ids = {subset["_id"]: subset for subset in subsets} + + sorted_versions = list(io.find({ + "type": "version", + "parent": {"$in": subset_entity_by_ids.keys()} + }).sort("name", -1)) + + subset_id_with_latest_version = [] + last_versions_by_id = {} + for version in sorted_versions: + subset_id = version["parent"] + if subset_id in subset_id_with_latest_version: + continue + subset_id_with_latest_version.append(subset_id) + last_versions_by_id[version["_id"]] = version + + repres = io.find({ + "type": "representation", + "parent": {"$in": last_versions_by_id.keys()} + }) + + output = {} + for repre in repres: + version_id = repre["parent"] + version = last_versions_by_id[version_id] + + subset_id = version["parent"] + subset = subset_entity_by_ids[subset_id] + + asset_id = subset["parent"] + asset = asset_entity_by_ids[asset_id] + + if asset_id not in output: + output[asset_id] = { + "asset_entity": asset, + "subsets": {} + } + + if subset_id not in output[asset_id]["subsets"]: + output[asset_id]["subsets"][subset_id] = { + "subset_entity": subset, + "version": { + "version_entity": version, + "repres": [] + } + } + + output[asset_id]["subsets"][subset_id]["version"]["repres"].append( + repre + ) + + return output diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index c559324a5e7..58c9abd71fb 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -327,653 +327,6 @@ def get_last_version_from_path(path_dir, filter): return None -class BuildWorkfile: - """Wrapper for build workfile process. - - Load representations for current context by build presets. Build presets - are host related, since each host has it's loaders. - """ - - log = logging.getLogger("BuildWorkfile") - - @staticmethod - def map_subsets_by_family(subsets): - subsets_by_family = collections.defaultdict(list) - for subset in subsets: - family = subset["data"].get("family") - if not family: - families = subset["data"].get("families") - if not families: - continue - family = families[0] - - subsets_by_family[family].append(subset) - return subsets_by_family - - def process(self): - """Main method of this wrapper. - - Building of workfile is triggered and is possible to implement - post processing of loaded containers if necessary. - """ - containers = self.build_workfile() - - return containers - - def build_workfile(self): - """Prepares and load containers into workfile. - - Loads latest versions of current and linked assets to workfile by logic - stored in Workfile profiles from presets. Profiles are set by host, - filtered by current task name and used by families. - - Each family can specify representation names and loaders for - representations and first available and successful loaded - representation is returned as container. - - At the end you'll get list of loaded containers per each asset. - - loaded_containers [{ - "asset_entity": , - "containers": [, , ...] - }, { - "asset_entity": , - "containers": [, ...] - }, { - ... - }] - """ - # Get current asset name and entity - current_asset_name = io.Session["AVALON_ASSET"] - current_asset_entity = io.find_one({ - "type": "asset", - "name": current_asset_name - }) - - # Skip if asset was not found - if not current_asset_entity: - print("Asset entity with name `{}` was not found".format( - current_asset_name - )) - return - - # Prepare available loaders - loaders_by_name = {} - for loader in avalon.api.discover(avalon.api.Loader): - loader_name = loader.__name__ - if loader_name in loaders_by_name: - raise KeyError( - "Duplicated loader name {0}!".format(loader_name) - ) - loaders_by_name[loader_name] = loader - - # Skip if there are any loaders - if not loaders_by_name: - self.log.warning("There are no registered loaders.") - return - - # Get current task name - current_task_name = io.Session["AVALON_TASK"] - - # Load workfile presets for task - self.build_presets = self.get_build_presets(current_task_name) - - # Skip if there are any presets for task - if not self.build_presets: - self.log.warning( - "Current task `{}` does not have any loading preset.".format( - current_task_name - ) - ) - return - - # Get presets for loading current asset - current_context_profiles = self.build_presets.get("current_context") - # Get presets for loading linked assets - link_context_profiles = self.build_presets.get("linked_assets") - # Skip if both are missing - if not current_context_profiles and not link_context_profiles: - self.log.warning( - "Current task `{}` has empty loading preset.".format( - current_task_name - ) - ) - return - - elif not current_context_profiles: - self.log.warning(( - "Current task `{}` doesn't have any loading" - " preset for it's context." - ).format(current_task_name)) - - elif not link_context_profiles: - self.log.warning(( - "Current task `{}` doesn't have any" - "loading preset for it's linked assets." - ).format(current_task_name)) - - # Prepare assets to process by workfile presets - assets = [] - current_asset_id = None - if current_context_profiles: - # Add current asset entity if preset has current context set - assets.append(current_asset_entity) - current_asset_id = current_asset_entity["_id"] - - if link_context_profiles: - # Find and append linked assets if preset has set linked mapping - link_assets = get_linked_assets(current_asset_entity) - if link_assets: - assets.extend(link_assets) - - # Skip if there are no assets. This can happen if only linked mapping - # is set and there are no links for his asset. - if not assets: - self.log.warning( - "Asset does not have linked assets. Nothing to process." - ) - return - - # Prepare entities from database for assets - prepared_entities = self._collect_last_version_repres(assets) - - # Load containers by prepared entities and presets - loaded_containers = [] - # - Current asset containers - if current_asset_id and current_asset_id in prepared_entities: - current_context_data = prepared_entities.pop(current_asset_id) - loaded_data = self.load_containers_by_asset_data( - current_context_data, current_context_profiles, loaders_by_name - ) - if loaded_data: - loaded_containers.append(loaded_data) - - # - Linked assets container - for linked_asset_data in prepared_entities.values(): - loaded_data = self.load_containers_by_asset_data( - linked_asset_data, link_context_profiles, loaders_by_name - ) - if loaded_data: - loaded_containers.append(loaded_data) - - # Return list of loaded containers - return loaded_containers - - def get_build_presets(self, task_name): - """ Returns presets to build workfile for task name. - - Presets are loaded for current project set in - io.Session["AVALON_PROJECT"], filtered by registered host - and entered task name. - - :param task_name: Task name used for filtering build presets. - :type task_name: str - :return: preset per eneter task - :rtype: dict | None - """ - host_name = avalon.api.registered_host().__name__.rsplit(".", 1)[-1] - presets = config.get_presets(io.Session["AVALON_PROJECT"]) - # Get presets for host - build_presets = ( - presets["plugins"] - .get(host_name, {}) - .get("workfile_build") - ) - if not build_presets: - return - - task_name_low = task_name.lower() - per_task_preset = None - for preset in build_presets: - preset_tasks = preset.get("tasks") or [] - preset_tasks_low = [task.lower() for task in preset_tasks] - if task_name_low in preset_tasks_low: - per_task_preset = preset - break - - return per_task_preset - - def _filter_build_profiles(self, build_profiles, loaders_by_name): - """ Filter build profiles by loaders and prepare process data. - - Valid profile must have "loaders", "families" and "repre_names" keys - with valid values. - - "loaders" expects list of strings representing possible loaders. - - "families" expects list of strings for filtering - by main subset family. - - "repre_names" expects list of strings for filtering by - representation name. - - Lowered "families" and "repre_names" are prepared for each profile with - all required keys. - - :param build_profiles: Profiles for building workfile. - :type build_profiles: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Filtered and prepared profiles. - :rtype: list - """ - valid_profiles = [] - for profile in build_profiles: - # Check loaders - profile_loaders = profile.get("loaders") - if not profile_loaders: - self.log.warning(( - "Build profile has missing loaders configuration: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Check if any loader is available - loaders_match = False - for loader_name in profile_loaders: - if loader_name in loaders_by_name: - loaders_match = True - break - - if not loaders_match: - self.log.warning(( - "All loaders from Build profile are not available: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Check families - profile_families = profile.get("families") - if not profile_families: - self.log.warning(( - "Build profile is missing families configuration: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Check representation names - profile_repre_names = profile.get("repre_names") - if not profile_repre_names: - self.log.warning(( - "Build profile is missing" - " representation names filtering: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Prepare lowered families and representation names - profile["families_lowered"] = [ - fam.lower() for fam in profile_families - ] - profile["repre_names_lowered"] = [ - name.lower() for name in profile_repre_names - ] - - valid_profiles.append(profile) - - return valid_profiles - - def _prepare_profile_for_subsets(self, subsets, profiles): - """Select profile for each subset byt it's data. - - Profiles are filtered for each subset individually. - Profile is filtered by subset's family, optionally by name regex and - representation names set in profile. - It is possible to not find matching profile for subset, in that case - subset is skipped and it is possible that none of subsets have - matching profile. - - :param subsets: Subset documents. - :type subsets: list - :param profiles: Build profiles. - :type profiles: dict - :return: Profile by subset's id. - :rtype: dict - """ - # Prepare subsets - subsets_by_family = self.map_subsets_by_family(subsets) - - profiles_per_subset_id = {} - for family, subsets in subsets_by_family.items(): - family_low = family.lower() - for profile in profiles: - # Skip profile if does not contain family - if family_low not in profile["families_lowered"]: - continue - - # Precompile name filters as regexes - profile_regexes = profile.get("subset_name_filters") - if profile_regexes: - _profile_regexes = [] - for regex in profile_regexes: - _profile_regexes.append(re.compile(regex)) - profile_regexes = _profile_regexes - - # TODO prepare regex compilation - for subset in subsets: - # Verify regex filtering (optional) - if profile_regexes: - valid = False - for pattern in profile_regexes: - if re.match(pattern, subset["name"]): - valid = True - break - - if not valid: - continue - - profiles_per_subset_id[subset["_id"]] = profile - - # break profiles loop on finding the first matching profile - break - return profiles_per_subset_id - - def load_containers_by_asset_data( - self, asset_entity_data, build_profiles, loaders_by_name - ): - """Load containers for entered asset entity by Build profiles. - - :param asset_entity_data: Prepared data with subsets, last version - and representations for specific asset. - :type asset_entity_data: dict - :param build_profiles: Build profiles. - :type build_profiles: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Output contains asset document and loaded containers. - :rtype: dict - """ - - # Make sure all data are not empty - if not asset_entity_data or not build_profiles or not loaders_by_name: - return - - asset_entity = asset_entity_data["asset_entity"] - - valid_profiles = self._filter_build_profiles( - build_profiles, loaders_by_name - ) - if not valid_profiles: - self.log.warning( - "There are not valid Workfile profiles. Skipping process." - ) - return - - self.log.debug("Valid Workfile profiles: {}".format(valid_profiles)) - - subsets_by_id = {} - version_by_subset_id = {} - repres_by_version_id = {} - for subset_id, in_data in asset_entity_data["subsets"].items(): - subset_entity = in_data["subset_entity"] - subsets_by_id[subset_entity["_id"]] = subset_entity - - version_data = in_data["version"] - version_entity = version_data["version_entity"] - version_by_subset_id[subset_id] = version_entity - repres_by_version_id[version_entity["_id"]] = ( - version_data["repres"] - ) - - if not subsets_by_id: - self.log.warning("There are not subsets for asset {0}".format( - asset_entity["name"] - )) - return - - profiles_per_subset_id = self._prepare_profile_for_subsets( - subsets_by_id.values(), valid_profiles - ) - if not profiles_per_subset_id: - self.log.warning("There are not valid subsets.") - return - - valid_repres_by_subset_id = collections.defaultdict(list) - for subset_id, profile in profiles_per_subset_id.items(): - profile_repre_names = profile["repre_names_lowered"] - - version_entity = version_by_subset_id[subset_id] - version_id = version_entity["_id"] - repres = repres_by_version_id[version_id] - for repre in repres: - repre_name_low = repre["name"].lower() - if repre_name_low in profile_repre_names: - valid_repres_by_subset_id[subset_id].append(repre) - - # DEBUG message - msg = "Valid representations for Asset: `{}`".format( - asset_entity["name"] - ) - for subset_id, repres in valid_repres_by_subset_id.items(): - subset = subsets_by_id[subset_id] - msg += "\n# Subset Name/ID: `{}`/{}".format( - subset["name"], subset_id - ) - for repre in repres: - msg += "\n## Repre name: `{}`".format(repre["name"]) - - self.log.debug(msg) - - containers = self._load_containers( - valid_repres_by_subset_id, subsets_by_id, - profiles_per_subset_id, loaders_by_name - ) - - return { - "asset_entity": asset_entity, - "containers": containers - } - - def _load_containers( - self, repres_by_subset_id, subsets_by_id, - profiles_per_subset_id, loaders_by_name - ): - """Real load by collected data happens here. - - Loading of representations per subset happens here. Each subset can - loads one representation. Loading is tried in specific order. - Representations are tried to load by names defined in configuration. - If subset has representation matching representation name each loader - is tried to load it until any is successful. If none of them was - successful then next reprensentation name is tried. - Subset process loop ends when any representation is loaded or - all matching representations were already tried. - - :param repres_by_subset_id: Available representations mapped - by their parent (subset) id. - :type repres_by_subset_id: dict - :param subsets_by_id: Subset documents mapped by their id. - :type subsets_by_id: dict - :param profiles_per_subset_id: Build profiles mapped by subset id. - :type profiles_per_subset_id: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Objects of loaded containers. - :rtype: list - """ - loaded_containers = [] - - # Get subset id order from build presets. - build_presets = self.build_presets.get("current_context", []) - build_presets += self.build_presets.get("linked_assets", []) - subset_ids_ordered = [] - for preset in build_presets: - for preset_family in preset["families"]: - for id, subset in subsets_by_id.items(): - if preset_family not in subset["data"].get("families", []): - continue - - subset_ids_ordered.append(id) - - # Order representations from subsets. - print("repres_by_subset_id", repres_by_subset_id) - representations_ordered = [] - representations = [] - for id in subset_ids_ordered: - for subset_id, repres in repres_by_subset_id.items(): - if repres in representations: - continue - - if id == subset_id: - representations_ordered.append((subset_id, repres)) - representations.append(repres) - - print("representations", representations) - - # Load ordered reprensentations. - for subset_id, repres in representations_ordered: - subset_name = subsets_by_id[subset_id]["name"] - - profile = profiles_per_subset_id[subset_id] - loaders_last_idx = len(profile["loaders"]) - 1 - repre_names_last_idx = len(profile["repre_names_lowered"]) - 1 - - repre_by_low_name = { - repre["name"].lower(): repre for repre in repres - } - - is_loaded = False - for repre_name_idx, profile_repre_name in enumerate( - profile["repre_names_lowered"] - ): - # Break iteration if representation was already loaded - if is_loaded: - break - - repre = repre_by_low_name.get(profile_repre_name) - if not repre: - continue - - for loader_idx, loader_name in enumerate(profile["loaders"]): - if is_loaded: - break - - loader = loaders_by_name.get(loader_name) - if not loader: - continue - try: - container = avalon.api.load( - loader, - repre["_id"], - name=subset_name - ) - loaded_containers.append(container) - is_loaded = True - - except Exception as exc: - if exc == pipeline.IncompatibleLoaderError: - self.log.info(( - "Loader `{}` is not compatible with" - " representation `{}`" - ).format(loader_name, repre["name"])) - - else: - self.log.error( - "Unexpected error happened during loading", - exc_info=True - ) - - msg = "Loading failed." - if loader_idx < loaders_last_idx: - msg += " Trying next loader." - elif repre_name_idx < repre_names_last_idx: - msg += ( - " Loading of subset `{}` was not successful." - ).format(subset_name) - else: - msg += " Trying next representation." - self.log.info(msg) - - return loaded_containers - - def _collect_last_version_repres(self, asset_entities): - """Collect subsets, versions and representations for asset_entities. - - :param asset_entities: Asset entities for which want to find data - :type asset_entities: list - :return: collected entities - :rtype: dict - - Example output: - ``` - { - {Asset ID}: { - "asset_entity": , - "subsets": { - {Subset ID}: { - "subset_entity": , - "version": { - "version_entity": , - "repres": [ - , , ... - ] - } - }, - ... - } - }, - ... - } - output[asset_id]["subsets"][subset_id]["version"]["repres"] - ``` - """ - - if not asset_entities: - return {} - - asset_entity_by_ids = {asset["_id"]: asset for asset in asset_entities} - - subsets = list(io.find({ - "type": "subset", - "parent": {"$in": asset_entity_by_ids.keys()} - })) - subset_entity_by_ids = {subset["_id"]: subset for subset in subsets} - - sorted_versions = list(io.find({ - "type": "version", - "parent": {"$in": subset_entity_by_ids.keys()} - }).sort("name", -1)) - - subset_id_with_latest_version = [] - last_versions_by_id = {} - for version in sorted_versions: - subset_id = version["parent"] - if subset_id in subset_id_with_latest_version: - continue - subset_id_with_latest_version.append(subset_id) - last_versions_by_id[version["_id"]] = version - - repres = io.find({ - "type": "representation", - "parent": {"$in": last_versions_by_id.keys()} - }) - - output = {} - for repre in repres: - version_id = repre["parent"] - version = last_versions_by_id[version_id] - - subset_id = version["parent"] - subset = subset_entity_by_ids[subset_id] - - asset_id = subset["parent"] - asset = asset_entity_by_ids[asset_id] - - if asset_id not in output: - output[asset_id] = { - "asset_entity": asset, - "subsets": {} - } - - if subset_id not in output[asset_id]["subsets"]: - output[asset_id]["subsets"][subset_id] = { - "subset_entity": subset, - "version": { - "version_entity": version, - "repres": [] - } - } - - output[asset_id]["subsets"][subset_id]["version"]["repres"].append( - repre - ) - - return output - - def ffprobe_streams(path_to_file, logger=None): """Load streams from entered filepath via ffprobe.""" if not logger: From ab621279276512463d8790b203571743fa2c1b9e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 11:57:08 +0100 Subject: [PATCH 36/47] ffprobe_streams moved to ffmpeg_utils --- pype/lib/__init__.py | 5 ++++- pype/lib/ffmpeg_utils.py | 40 ++++++++++++++++++++++++++++++++++++++++ pype/lib/lib_old.py | 33 --------------------------------- 3 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 pype/lib/ffmpeg_utils.py diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index f807fe894a5..25589fa84fc 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -43,6 +43,7 @@ ffprobe_streams, source_hash, ) +from .ffmpeg_utils import ffprobe_streams __all__ = [ "get_avalon_database", @@ -63,5 +64,7 @@ "launch_application", "ApplicationAction", - "filter_pyblish_plugins" + "filter_pyblish_plugins", + + "ffprobe_streams" ] diff --git a/pype/lib/ffmpeg_utils.py b/pype/lib/ffmpeg_utils.py new file mode 100644 index 00000000000..1c656d55d39 --- /dev/null +++ b/pype/lib/ffmpeg_utils.py @@ -0,0 +1,40 @@ +import logging +import json +import subprocess + +from . import get_ffmpeg_tool_path + +log = logging.getLogger("FFmpeg utils") + + +def ffprobe_streams(path_to_file, logger=None): + """Load streams from entered filepath via ffprobe.""" + if not logger: + logger = log + logger.info( + "Getting information about input \"{}\".".format(path_to_file) + ) + args = [ + "\"{}\"".format(get_ffmpeg_tool_path("ffprobe")), + "-v quiet", + "-print_format json", + "-show_format", + "-show_streams", + "\"{}\"".format(path_to_file) + ] + command = " ".join(args) + logger.debug("FFprobe command: \"{}\"".format(command)) + popen = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + popen_stdout, popen_stderr = popen.communicate() + if popen_stdout: + logger.debug("ffprobe stdout: {}".format(popen_stdout)) + + if popen_stderr: + logger.debug("ffprobe stderr: {}".format(popen_stderr)) + return json.loads(popen_stdout)["streams"] diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 58c9abd71fb..37cd6d8f93f 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -327,39 +327,6 @@ def get_last_version_from_path(path_dir, filter): return None -def ffprobe_streams(path_to_file, logger=None): - """Load streams from entered filepath via ffprobe.""" - if not logger: - logger = log - logger.info( - "Getting information about input \"{}\".".format(path_to_file) - ) - args = [ - "\"{}\"".format(get_ffmpeg_tool_path("ffprobe")), - "-v quiet", - "-print_format json", - "-show_format", - "-show_streams", - "\"{}\"".format(path_to_file) - ] - command = " ".join(args) - logger.debug("FFprobe command: \"{}\"".format(command)) - popen = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - - popen_stdout, popen_stderr = popen.communicate() - if popen_stdout: - logger.debug("ffprobe stdout: {}".format(popen_stdout)) - - if popen_stderr: - logger.debug("ffprobe stderr: {}".format(popen_stderr)) - return json.loads(popen_stdout)["streams"] - - def source_hash(filepath, *args): """Generate simple identifier for a source file. This is used to identify whether a source file has previously been From e153fbef3686e8778d1303a91fd3a24c23a667ae Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 11:58:34 +0100 Subject: [PATCH 37/47] cleaned up imports --- pype/lib/lib_old.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py index 37cd6d8f93f..89b7f42d385 100644 --- a/pype/lib/lib_old.py +++ b/pype/lib/lib_old.py @@ -1,15 +1,11 @@ import os import re -import json -import collections import logging import itertools import contextlib import subprocess -from avalon import io, pipeline import avalon.api -from ..api import config log = logging.getLogger(__name__) From dafa84e2a49ba5f0c3d45e3d9f8d9c5694a28118 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 12:16:17 +0100 Subject: [PATCH 38/47] removed `get_subsets` from pype.api --- pype/api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/api.py b/pype/api.py index 2c7dfa73f07..29e91dc8e0c 100644 --- a/pype/api.py +++ b/pype/api.py @@ -40,7 +40,6 @@ version_up, get_asset, get_hierarchy, - get_subsets, get_version_from_path, get_last_version_from_path, modified_environ, @@ -89,7 +88,6 @@ "version_up", "get_hierarchy", "get_asset", - "get_subsets", "get_version_from_path", "get_last_version_from_path", "modified_environ", From 79cf8783634972ed8817baf9b40f364fd86fcbd0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 13:20:56 +0100 Subject: [PATCH 39/47] removed imports from old lib --- pype/lib/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 25589fa84fc..b5a819653b2 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -39,8 +39,6 @@ _get_host_name, get_version_from_path, get_last_version_from_path, - BuildWorkfile, - ffprobe_streams, source_hash, ) from .ffmpeg_utils import ffprobe_streams From f591d76367db917448de46e2676e38c09b81b421 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 13:21:17 +0100 Subject: [PATCH 40/47] hound fixes --- pype/plugins/celaction/publish/collect_audio.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pype/plugins/celaction/publish/collect_audio.py b/pype/plugins/celaction/publish/collect_audio.py index c92e4fd868b..341db4250eb 100644 --- a/pype/plugins/celaction/publish/collect_audio.py +++ b/pype/plugins/celaction/publish/collect_audio.py @@ -45,9 +45,9 @@ def process(self, context): def get_subsets( self, asset_name, + representations, regex_filter=None, - version=None, - representations=["exr", "dpx"] + version=None ): """ Query subsets with filter on name. @@ -99,7 +99,9 @@ def get_subsets( sort=[("name", -1)] ) else: - assert isinstance(version, int), "version needs to be `int` type" + assert isinstance(version, int), ( + "version needs to be `int` type" + ) version_sel = io.find_one({ "type": "version", "parent": subset["_id"], From e933955dc8bd3b516736a53f505bb08d32cec76f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 11 Nov 2020 18:57:17 +0100 Subject: [PATCH 41/47] #664 - added tests for #697 --- pype/tests/test_lib_restructuralization.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py index 92197c82326..d8ef4f2f1e6 100644 --- a/pype/tests/test_lib_restructuralization.py +++ b/pype/tests/test_lib_restructuralization.py @@ -16,5 +16,11 @@ def test_backward_compatibility(printer): from pype.lib import get_avalon_database from pype.lib import set_io_database + from pype.lib import get_ffmpeg_tool_path + from pype.lib import get_last_version_from_path + from pype.lib import get_paths_from_environ + from pype.lib import get_version_from_path + from pype.lib import version_up + except ImportError as e: raise From 7f97e41b041d025ca7758ba666d6871b51ef3441 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 19:45:07 +0100 Subject: [PATCH 42/47] `get_subsets` is more efficient now --- .../celaction/publish/collect_audio.py | 134 +++++++++--------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/pype/plugins/celaction/publish/collect_audio.py b/pype/plugins/celaction/publish/collect_audio.py index 341db4250eb..c515dd58a3d 100644 --- a/pype/plugins/celaction/publish/collect_audio.py +++ b/pype/plugins/celaction/publish/collect_audio.py @@ -1,4 +1,5 @@ import os +import collections import pyblish.api from avalon import io @@ -14,12 +15,13 @@ class AppendCelactionAudio(pyblish.api.ContextPlugin): def process(self, context): self.log.info('Collecting Audio Data') - asset_entity = context.data["assetEntity"] + asset_doc = context.data["assetEntity"] # get all available representations - subsets = self.get_subsets(asset_entity["name"], - representations=["audio", "wav"] - ) + subsets = self.get_subsets( + asset_doc, + representations=["audio", "wav"] + ) self.log.info(f"subsets is: {pformat(subsets)}") if not subsets.get("audioMain"): @@ -42,13 +44,7 @@ def process(self, context): else: self.log.warning("Couldn't find any audio file on Ftrack.") - def get_subsets( - self, - asset_name, - representations, - regex_filter=None, - version=None - ): + def get_subsets(self, asset_doc, representations): """ Query subsets with filter on name. @@ -57,67 +53,75 @@ def get_subsets( can be filtered. Arguments: - asset_name (str): asset (shot) name - regex_filter (raw): raw string with filter pattern - version (str or int): `last` or number of version + asset_doct (dict): Asset (shot) mongo document representations (list): list for all representations Returns: dict: subsets with version and representaions in keys """ - # query asset from db - asset_io = io.find_one({"type": "asset", "name": asset_name}) - - # check if anything returned - assert asset_io, ( - "Asset not existing. Check correct name: `{}`").format(asset_name) - - # create subsets query filter - filter_query = {"type": "subset", "parent": asset_io["_id"]} - - # add reggex filter string into query filter - if regex_filter: - filter_query["name"] = {"$regex": r"{}".format(regex_filter)} - - # query all assets - subsets = list(io.find(filter_query)) - - assert subsets, ("No subsets found. Check correct filter. " - "Try this for start `r'.*'`: " - "asset: `{}`").format(asset_name) + # Query all subsets for asset + subset_docs = io.find({ + "type": "subset", + "parent": asset_doc["_id"] + }) + # Collect all subset ids + subset_ids = [ + subset_doc["_id"] + for subset_doc in subset_docs + ] + + # Check if we found anything + assert subset_ids, ( + "No subsets found. Check correct filter. " + "Try this for start `r'.*'`: asset: `{}`" + ).format(asset_doc["name"]) + + # Last version aggregation + pipeline = [ + # Find all versions of those subsets + {"$match": { + "type": "version", + "parent": {"$in": subset_ids} + }}, + # Sorting versions all together + {"$sort": {"name": 1}}, + # Group them by "parent", but only take the last + {"$group": { + "_id": "$parent", + "_version_id": {"$last": "$_id"}, + "name": {"$last": "$name"} + }} + ] + last_versions_by_subset_id = dict() + for doc in io.aggregate(pipeline): + doc["parent"] = doc["_id"] + doc["_id"] = doc.pop("_version_id") + last_versions_by_subset_id[doc["parent"]] = doc + + version_docs_by_id = {} + for version_doc in last_versions_by_subset_id.values(): + version_docs_by_id[version_doc["_id"]] = version_doc + + repre_docs = io.find({ + "type": "representation", + "parent": {"$in": list(version_docs_by_id.keys())}, + "name": {"$in": representations} + }) + repre_docs_by_version_id = collections.defaultdict(list) + for repre_doc in repre_docs: + version_id = repre_doc["parent"] + repre_docs_by_version_id[version_id].append(repre_doc) output_dict = {} - # Process subsets - for subset in subsets: - if not version: - version_sel = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - else: - assert isinstance(version, int), ( - "version needs to be `int` type" - ) - version_sel = io.find_one({ - "type": "version", - "parent": subset["_id"], - "name": int(version) - }) - - find_dict = {"type": "representation", - "parent": version_sel["_id"]} - - filter_repr = {"name": {"$in": representations}} - - find_dict.update(filter_repr) - repres_out = [i for i in io.find(find_dict)] - - if len(repres_out) > 0: - output_dict[subset["name"]] = {"version": version_sel, - "representations": repres_out} + for version_id, repre_docs in repre_docs_by_version_id.items(): + version_doc = version_docs_by_id[version_id] + subset_id = version_doc["parent"] + subset_doc = last_versions_by_subset_id[subset_id] + # Store queried docs by subset name + output_dict[subset_doc["name"]] = { + "representations": repre_docs, + "version": version_doc + } return output_dict From 5d07af223cccd5b2b79a654171be648811531825 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 11 Nov 2020 20:25:44 +0100 Subject: [PATCH 43/47] #664 - moved last 2 functions removed obsolete lib_old.py --- pype/lib/__init__.py | 14 ++-- pype/lib/applications.py | 66 +++++++++++++++ pype/lib/lib_old.py | 94 ---------------------- pype/lib/plugin_tools.py | 21 +++++ pype/tests/test_lib_restructuralization.py | 3 + 5 files changed, 97 insertions(+), 101 deletions(-) delete mode 100644 pype/lib/lib_old.py diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index bcc0d352e64..1ade97cd0e3 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -21,10 +21,11 @@ from .applications import ( ApplicationLaunchFailed, launch_application, - ApplicationAction + ApplicationAction, + _subprocess ) -from .plugin_tools import filter_pyblish_plugins +from .plugin_tools import filter_pyblish_plugins, source_hash from .path_tools import ( version_up, @@ -34,10 +35,6 @@ get_ffmpeg_tool_path ) -from .lib_old import ( - _subprocess, - source_hash -) from .ffmpeg_utils import ffprobe_streams __all__ = [ @@ -67,5 +64,8 @@ "get_paths_from_environ", "get_ffmpeg_tool_path", - "ffprobe_streams" + "ffprobe_streams", + + "source_hash", + "_subprocess" ] diff --git a/pype/lib/applications.py b/pype/lib/applications.py index fd3d0ef9906..159840ceb5d 100644 --- a/pype/lib/applications.py +++ b/pype/lib/applications.py @@ -4,6 +4,7 @@ import copy import platform import logging +import subprocess import acre @@ -389,3 +390,68 @@ def _start_timer(self, session, entity, _ftrack_api): on_error="ignore" ) self.log.debug("Timer start triggered successfully.") + + +# Special naming case for subprocess since its a built-in method. +def _subprocess(*args, **kwargs): + """Convenience method for getting output errors for subprocess. + + Entered arguments and keyword arguments are passed to subprocess Popen. + + Args: + *args: Variable length arument list passed to Popen. + **kwargs : Arbitary keyword arguments passed to Popen. Is possible to + pass `logging.Logger` object under "logger" if want to use + different than lib's logger. + + Returns: + str: Full output of subprocess concatenated stdout and stderr. + + Raises: + RuntimeError: Exception is raised if process finished with nonzero + return code. + """ + + # Get environents from kwarg or use current process environments if were + # not passed. + env = kwargs.get("env") or os.environ + # Make sure environment contains only strings + filtered_env = {k: str(v) for k, v in env.items()} + + # Use lib's logger if was not passed with kwargs. + logger = kwargs.pop("logger", log) + + # set overrides + kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) + kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) + kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) + kwargs['env'] = filtered_env + + proc = subprocess.Popen(*args, **kwargs) + + full_output = "" + _stdout, _stderr = proc.communicate() + if _stdout: + _stdout = _stdout.decode("utf-8") + full_output += _stdout + logger.debug(_stdout) + + if _stderr: + _stderr = _stderr.decode("utf-8") + # Add additional line break if output already containt stdout + if full_output: + full_output += "\n" + full_output += _stderr + logger.warning(_stderr) + + if proc.returncode != 0: + exc_msg = "Executing arguments was not successful: \"{}\"".format(args) + if _stdout: + exc_msg += "\n\nOutput:\n{}".format(_stdout) + + if _stderr: + exc_msg += "Error:\n{}".format(_stderr) + + raise RuntimeError(exc_msg) + + return full_output diff --git a/pype/lib/lib_old.py b/pype/lib/lib_old.py deleted file mode 100644 index be4211d0677..00000000000 --- a/pype/lib/lib_old.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import logging -import contextlib -import subprocess - -import avalon.api - -log = logging.getLogger(__name__) - - -# Special naming case for subprocess since its a built-in method. -def _subprocess(*args, **kwargs): - """Convenience method for getting output errors for subprocess. - - Entered arguments and keyword arguments are passed to subprocess Popen. - - Args: - *args: Variable length arument list passed to Popen. - **kwargs : Arbitary keyword arguments passed to Popen. Is possible to - pass `logging.Logger` object under "logger" if want to use - different than lib's logger. - - Returns: - str: Full output of subprocess concatenated stdout and stderr. - - Raises: - RuntimeError: Exception is raised if process finished with nonzero - return code. - """ - - # Get environents from kwarg or use current process environments if were - # not passed. - env = kwargs.get("env") or os.environ - # Make sure environment contains only strings - filtered_env = {k: str(v) for k, v in env.items()} - - # Use lib's logger if was not passed with kwargs. - logger = kwargs.pop("logger", log) - - # set overrides - kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) - kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) - kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) - kwargs['env'] = filtered_env - - proc = subprocess.Popen(*args, **kwargs) - - full_output = "" - _stdout, _stderr = proc.communicate() - if _stdout: - _stdout = _stdout.decode("utf-8") - full_output += _stdout - logger.debug(_stdout) - - if _stderr: - _stderr = _stderr.decode("utf-8") - # Add additional line break if output already containt stdout - if full_output: - full_output += "\n" - full_output += _stderr - logger.warning(_stderr) - - if proc.returncode != 0: - exc_msg = "Executing arguments was not successful: \"{}\"".format(args) - if _stdout: - exc_msg += "\n\nOutput:\n{}".format(_stdout) - - if _stderr: - exc_msg += "Error:\n{}".format(_stderr) - - raise RuntimeError(exc_msg) - - return full_output - - -def source_hash(filepath, *args): - """Generate simple identifier for a source file. - This is used to identify whether a source file has previously been - processe into the pipeline, e.g. a texture. - The hash is based on source filepath, modification time and file size. - This is only used to identify whether a specific source file was already - published before from the same location with the same modification date. - We opt to do it this way as opposed to Avalanch C4 hash as this is much - faster and predictable enough for all our production use cases. - Args: - filepath (str): The source file path. - You can specify additional arguments in the function - to allow for specific 'processing' values to be included. - """ - # We replace dots with comma because . cannot be a key in a pymongo dict. - file_name = os.path.basename(filepath) - time = str(os.path.getmtime(filepath)) - size = str(os.path.getsize(filepath)) - return "|".join([file_name, time, size] + list(args)).replace(".", ",") diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py index 066f1ff20a5..0b6ace807eb 100644 --- a/pype/lib/plugin_tools.py +++ b/pype/lib/plugin_tools.py @@ -57,3 +57,24 @@ def filter_pyblish_plugins(plugins): option, value, plugin.__name__)) setattr(plugin, option, value) + + +def source_hash(filepath, *args): + """Generate simple identifier for a source file. + This is used to identify whether a source file has previously been + processe into the pipeline, e.g. a texture. + The hash is based on source filepath, modification time and file size. + This is only used to identify whether a specific source file was already + published before from the same location with the same modification date. + We opt to do it this way as opposed to Avalanch C4 hash as this is much + faster and predictable enough for all our production use cases. + Args: + filepath (str): The source file path. + You can specify additional arguments in the function + to allow for specific 'processing' values to be included. + """ + # We replace dots with comma because . cannot be a key in a pymongo dict. + file_name = os.path.basename(filepath) + time = str(os.path.getmtime(filepath)) + size = str(os.path.getsize(filepath)) + return "|".join([file_name, time, size] + list(args)).replace(".", ",") diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py index 5980f934c9a..152be8d1eb6 100644 --- a/pype/tests/test_lib_restructuralization.py +++ b/pype/tests/test_lib_restructuralization.py @@ -32,5 +32,8 @@ def test_backward_compatibility(printer): from pype.hosts.fusion.lib import switch_item + from pype.lib import source_hash + from pype.lib import _subprocess + except ImportError as e: raise From 433105ebac9e0751c09288d72d5888d9d553dea6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Nov 2020 10:22:13 +0100 Subject: [PATCH 44/47] #664 - removed obsolete Aport action --- docs/source/pype.aport.rst | 20 --------- docs/source/pype.rst | 1 - pype/plugins/launcher/actions/Aport.py | 61 -------------------------- 3 files changed, 82 deletions(-) delete mode 100644 docs/source/pype.aport.rst delete mode 100644 pype/plugins/launcher/actions/Aport.py diff --git a/docs/source/pype.aport.rst b/docs/source/pype.aport.rst deleted file mode 100644 index 4a96b1e619a..00000000000 --- a/docs/source/pype.aport.rst +++ /dev/null @@ -1,20 +0,0 @@ -pype.aport package -================== - -.. automodule:: pype.aport - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -pype.aport.api module ---------------------- - -.. automodule:: pype.aport.api - :members: - :undoc-members: - :show-inheritance: - - diff --git a/docs/source/pype.rst b/docs/source/pype.rst index 7409ee62eee..a480cc31806 100644 --- a/docs/source/pype.rst +++ b/docs/source/pype.rst @@ -11,7 +11,6 @@ Subpackages .. toctree:: - pype.aport pype.avalon_apps pype.clockify pype.ftrack diff --git a/pype/plugins/launcher/actions/Aport.py b/pype/plugins/launcher/actions/Aport.py deleted file mode 100644 index 0ecd07c49a5..00000000000 --- a/pype/plugins/launcher/actions/Aport.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import acre - -from avalon import api, lib -import pype.api as pype -from pype.aport import lib as aportlib - -log = pype.Logger().get_logger(__name__, "aport") - - -class Aport(api.Action): - - name = "aport" - label = "Aport - Avalon's Server" - icon = "retweet" - order = 996 - - def is_compatible(self, session): - """Return whether the action is compatible with the session""" - if "AVALON_TASK" in session: - return True - return False - - def process(self, session, **kwargs): - """Implement the behavior for when the action is triggered - - Args: - session (dict): environment dictionary - - Returns: - Popen instance of newly spawned process - - """ - - with pype.modified_environ(**session): - # Get executable by name - print(self.name) - app = lib.get_application(self.name) - executable = lib.which(app["executable"]) - - # Run as server - arguments = [] - - tools_env = acre.get_tools([self.name]) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(os.environ)) - - if not env.get('AVALON_WORKDIR', None): - os.environ["AVALON_WORKDIR"] = aportlib.get_workdir_template() - - env.update(dict(os.environ)) - - try: - lib.launch( - executable=executable, - args=arguments, - environment=env - ) - except Exception as e: - log.error(e) - return From c1cbf3199dee7fd5513dcedff5f5d5d8f219cae7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Nov 2020 11:21:14 +0100 Subject: [PATCH 45/47] Updated docstrings --- pype/lib/avalon_context.py | 104 ++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/pype/lib/avalon_context.py b/pype/lib/avalon_context.py index 56abc4aee6f..6cecdb93e38 100644 --- a/pype/lib/avalon_context.py +++ b/pype/lib/avalon_context.py @@ -67,7 +67,16 @@ def any_outdated(): def get_asset(asset_name=None): - """ Returning asset document from database """ + """ Returning asset document from database by its name. + + Doesn't count with duplicities on asset names! + + Args: + asset_name (str) + + Returns: + (MongoDB document) + """ if not asset_name: asset_name = avalon.api.Session["AVALON_ASSET"] @@ -86,8 +95,11 @@ def get_hierarchy(asset_name=None): """ Obtain asset hierarchy path string from mongo db + Args: + asset_name (str) + Returns: - string: asset hierarchy path + (string): asset hierarchy path """ if not asset_name: @@ -127,7 +139,14 @@ def get_hierarchy(asset_name=None): def get_linked_assets(asset_entity): - """Return linked assets for `asset_entity`.""" + """Return linked assets for `asset_entity` from DB + + Args: + asset_entity (dict): asset document from DB + + Returns: + (list) of MongoDB documents + """ inputs = asset_entity["data"].get("inputs", []) inputs = [io.find_one({"_id": x}) for x in inputs] return inputs @@ -384,10 +403,11 @@ def get_build_presets(self, task_name): io.Session["AVALON_PROJECT"], filtered by registered host and entered task name. - :param task_name: Task name used for filtering build presets. - :type task_name: str - :return: preset per eneter task - :rtype: dict | None + Args: + task_name (str): Task name used for filtering build presets. + + Returns: + (dict): preset per entered task name """ host_name = avalon.api.registered_host().__name__.rsplit(".", 1)[-1] presets = config.get_presets(io.Session["AVALON_PROJECT"]) @@ -425,12 +445,12 @@ def _filter_build_profiles(self, build_profiles, loaders_by_name): Lowered "families" and "repre_names" are prepared for each profile with all required keys. - :param build_profiles: Profiles for building workfile. - :type build_profiles: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Filtered and prepared profiles. - :rtype: list + Args: + build_profiles (dict): Profiles for building workfile. + loaders_by_name (dict): Available loaders per name. + + Returns: + (list): Filtered and prepared profiles. """ valid_profiles = [] for profile in build_profiles: @@ -494,12 +514,12 @@ def _prepare_profile_for_subsets(self, subsets, profiles): subset is skipped and it is possible that none of subsets have matching profile. - :param subsets: Subset documents. - :type subsets: list - :param profiles: Build profiles. - :type profiles: dict - :return: Profile by subset's id. - :rtype: dict + Args: + subsets (list): Subset documents. + profiles (dict): Build profiles. + + Returns: + (dict) Profile by subset's id. """ # Prepare subsets subsets_by_family = self.map_subsets_by_family(subsets) @@ -544,15 +564,14 @@ def load_containers_by_asset_data( ): """Load containers for entered asset entity by Build profiles. - :param asset_entity_data: Prepared data with subsets, last version - and representations for specific asset. - :type asset_entity_data: dict - :param build_profiles: Build profiles. - :type build_profiles: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Output contains asset document and loaded containers. - :rtype: dict + Args: + asset_entity_data (dict): Prepared data with subsets, last version + and representations for specific asset. + build_profiles (dict): Build profiles. + loaders_by_name (dict): Available loaders per name. + + Returns: + (dict) Output contains asset document and loaded containers. """ # Make sure all data are not empty @@ -650,17 +669,15 @@ def _load_containers( Subset process loop ends when any representation is loaded or all matching representations were already tried. - :param repres_by_subset_id: Available representations mapped - by their parent (subset) id. - :type repres_by_subset_id: dict - :param subsets_by_id: Subset documents mapped by their id. - :type subsets_by_id: dict - :param profiles_per_subset_id: Build profiles mapped by subset id. - :type profiles_per_subset_id: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Objects of loaded containers. - :rtype: list + Args: + repres_by_subset_id (dict): Available representations mapped + by their parent (subset) id. + subsets_by_id (dict): Subset documents mapped by their id. + profiles_per_subset_id (dict): Build profiles mapped by subset id. + loaders_by_name (dict): Available loaders per name. + + Returns: + (list) Objects of loaded containers. """ loaded_containers = [] @@ -760,10 +777,11 @@ def _load_containers( def _collect_last_version_repres(self, asset_entities): """Collect subsets, versions and representations for asset_entities. - :param asset_entities: Asset entities for which want to find data - :type asset_entities: list - :return: collected entities - :rtype: dict + Args: + asset_entities (list): Asset entities for which want to find data + + Returns: + (dict): collected entities Example output: ``` From fd8eb5060d182cabe79f2faa9fd304bbaf3e85d1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Nov 2020 11:22:56 +0100 Subject: [PATCH 46/47] Updated docstrings --- pype/lib/ffmpeg_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/lib/ffmpeg_utils.py b/pype/lib/ffmpeg_utils.py index 1c656d55d39..0096fd13aa9 100644 --- a/pype/lib/ffmpeg_utils.py +++ b/pype/lib/ffmpeg_utils.py @@ -8,7 +8,13 @@ def ffprobe_streams(path_to_file, logger=None): - """Load streams from entered filepath via ffprobe.""" + """Load streams from entered filepath via ffprobe. + + Args: + path_to_file (str): absolute path + logger (logging.getLogger): injected logger, if empty new is created + + """ if not logger: logger = log logger.info( From 98e0d65040bba353cdbb23f9d7b4d1bb9ef59bdf Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Nov 2020 11:26:00 +0100 Subject: [PATCH 47/47] Updated docstrings --- pype/lib/path_tools.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pype/lib/path_tools.py b/pype/lib/path_tools.py index 11fc3bf8a64..ba383ea3eda 100644 --- a/pype/lib/path_tools.py +++ b/pype/lib/path_tools.py @@ -8,10 +8,12 @@ def get_paths_from_environ(env_key, return_first=False): """Return existing paths from specific envirnment variable. - :param env_key: Environment key where should look for paths. - :type env_key: str - :param return_first: Return first path on `True`, list of all on `False`. - :type return_first: boolean + Args: + env_key (str): Environment key where should look for paths. + + Returns: + (bool): Return first path on `True`, list of all on `False`. + Difference when none of paths exists: - when `return_first` is set to `False` then function returns empty list. @@ -47,8 +49,12 @@ def get_ffmpeg_tool_path(tool="ffmpeg"): Function looks for tool in paths set in FFMPEG_PATH environment. If tool exists then returns it's full path. - Returns tool name itself when tool path was not found. (FFmpeg path may be - set in PATH environment variable) + Args: + tool (string): tool name + + Returns: + (str): tool name itself when tool path was not found. (FFmpeg path + may be set in PATH environment variable) """ dir_paths = get_paths_from_environ("FFMPEG_PATH") for dir_path in dir_paths: @@ -70,8 +76,11 @@ def version_up(filepath): Parses for a version identifier like `_v001` or `.v001` When no version present _v001 is appended as suffix. + Args: + filepath (str): full url + Returns: - str: filepath with increased version number + (str): filepath with increased version number """ dirname = os.path.dirname(filepath)