diff --git a/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_cache_farm.py b/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_cache_farm.py index c558f35208..b7c3b55cae 100644 --- a/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_cache_farm.py +++ b/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_cache_farm.py @@ -27,23 +27,30 @@ def process(self, instance): # Why do we need this particular collector to collect the expected # output files from a ROP node. Don't we have a dedicated collector # for that yet? + # Answer: No, we don't have a generic expected file collector. + # Because different product types needs different logic. + # e.g. check CollectMantraROPRenderProducts + # and CollectKarmaROPRenderProducts # Collect expected files ropnode = hou.node(instance.data["instance_node"]) output_parm = lib.get_output_parameter(ropnode) expected_filepath = output_parm.eval() instance.data.setdefault("files", list()) instance.data.setdefault("expectedFiles", list()) - if instance.data.get("frames"): - files = self.get_files(instance, expected_filepath) - # list of files - instance.data["files"].extend(files) - else: + + frames = instance.data.get("frames", "") + if isinstance(frames, str): # single file - instance.data["files"].append(output_parm.eval()) - cache_files = {"_": instance.data["files"]} - # Convert instance family to pointcache if it is bgeo or abc - # because ??? - self.log.debug(instance.data["families"]) + instance.data["files"].append(expected_filepath) + else: + # list of files + staging_dir, _ = os.path.split(expected_filepath) + instance.data["files"].extend( + ["{}/{}".format(staging_dir, f) for f in frames] + ) + + cache_files = {"cache": instance.data["files"]} + instance.data.update({ "plugin": "Houdini", "publish": True @@ -51,23 +58,4 @@ def process(self, instance): instance.data["families"].append("publish.hou") instance.data["expectedFiles"].append(cache_files) - self.log.debug("{}".format(instance.data)) - - def get_files(self, instance, output_parm): - """Get the files with the frame range data - - Args: - instance (_type_): instance - output_parm (_type_): path of output parameter - - Returns: - files: a list of files - """ - directory = os.path.dirname(output_parm) - - files = [ - os.path.join(directory, frame).replace("\\", "/") - for frame in instance.data["frames"] - ] - - return files + self.log.debug("Caching on farm expected files: {}".format(instance.data["expectedFiles"])) diff --git a/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_files_for_cleaning_up.py b/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_files_for_cleaning_up.py new file mode 100644 index 0000000000..3ab03babf4 --- /dev/null +++ b/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_files_for_cleaning_up.py @@ -0,0 +1,98 @@ +import os +from typing import List + +import pyblish.api +from ayon_core.pipeline import AYONPyblishPluginMixin +from ayon_houdini.api import plugin + + +class CollectFilesForCleaningUp(plugin.HoudiniInstancePlugin, + AYONPyblishPluginMixin): + """Collect Files For Cleaning Up. + + This collector collects output files and adds them to file remove list. + + CAUTION: + This collector registers exported files and + the parent folder for deletion in `ExplicitCleanUp` plug-in. + please refer to `ExplicitCleanUp`'s docstring for further info. + + Notes: + Artists are free to change the file path in the ROP node. + + Farm instances will be processed on farm by other dedicated plugins + that live in core addon e.g. `CollectRenderedFiles` plugin. + These dedicated plugins don't support tracking and removing + intermediate render files. + + Local Render instances don't track intermediate render files, + Therefore, this plugin doesn't support removing + intermediate render files. + + HDA is not added to this plugin's options in server settings. + Cleaning up HDA products will break the scene as Houdini will no longer + be able to find the HDA file. + In addition,HDA plugins always save HDAs to external files. + Therefore, Cleaning up HDA products will break the ability to go back + to the workfile and continue on the HDA. + """ + + # It should run after CollectFrames and Collect Render plugins, + # and before CollectLocalRenderInstances. + order = pyblish.api.CollectorOrder + 0.115 + + hosts = ["houdini"] + families = ["*"] + label = "Collect Files For Cleaning Up" + + def process(self, instance): + + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return + + files: List[str] = [] + staging_dirs: List[str] = [] + expected_files = instance.data.get("expectedFiles", []) + + # Prefer 'expectedFiles' over 'frames' because it usually contains more + # output files than just a single file or single sequence of files. + if expected_files: + # Products with expected files + # This can be Render products or submitted cache to farm. + for expected in expected_files: + # expected.values() is a list of lists + for output_files in expected.values(): + staging_dir, _ = os.path.split(output_files[0]) + if staging_dir not in staging_dirs: + staging_dirs.append(staging_dir) + files.extend(output_files) + else: + # Products with frames or single file. + + frames = instance.data.get("frames") + if frames is None: + self.log.warning( + f"No frames data found on instance {instance}" + ". Skipping collection for caching on farm..." + ) + return + + staging_dir = instance.data.get("stagingDir") + staging_dirs.append(staging_dir) + + if isinstance(frames, str): + # single file. + files.append(f"{staging_dir}/{frames}") + else: + # list of frame. + files.extend( + [f"{staging_dir}/{frame}" for frame in frames] + ) + + self.log.debug( + f"Add directories to 'cleanupEmptyDir': {staging_dirs}") + instance.context.data["cleanupEmptyDirs"].extend(staging_dirs) + + self.log.debug("Add files to 'cleanupFullPaths': {}".format(files)) + instance.context.data["cleanupFullPaths"].extend(files) diff --git a/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_frames.py b/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_frames.py index 01dd5fdf05..a442e74835 100644 --- a/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_frames.py +++ b/server_addon/houdini/client/ayon_houdini/plugins/publish/collect_frames.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- """Collector plugin for frames data on ROP instances.""" import os -import re - import hou # noqa +import clique import pyblish.api from ayon_houdini.api import lib, plugin @@ -16,86 +15,49 @@ class CollectFrames(plugin.HoudiniInstancePlugin): order = pyblish.api.CollectorOrder + 0.1 label = "Collect Frames" families = ["camera", "vdbcache", "imagesequence", "ass", - "redshiftproxy", "review", "pointcache", "fbx"] + "redshiftproxy", "review", "pointcache", "fbx", + "model"] def process(self, instance): - ropnode = hou.node(instance.data["instance_node"]) - - start_frame = instance.data.get("frameStartHandle", None) - end_frame = instance.data.get("frameEndHandle", None) + # CollectRopFrameRange computes `start_frame` and `end_frame` + # depending on the trange value. + start_frame = instance.data["frameStartHandle"] + end_frame = instance.data["frameEndHandle"] + # Evaluate the file name at the first frame. + ropnode = hou.node(instance.data["instance_node"]) output_parm = lib.get_output_parameter(ropnode) - if start_frame is not None: - # When rendering only a single frame still explicitly - # get the name for that particular frame instead of current frame - output = output_parm.evalAtFrame(start_frame) - else: - self.log.warning("Using current frame: {}".format(hou.frame())) - output = output_parm.eval() - - _, ext = lib.splitext( - output, allowed_multidot_extensions=[ - ".ass.gz", ".bgeo.sc", ".bgeo.gz", - ".bgeo.lzma", ".bgeo.bz2"]) + output = output_parm.evalAtFrame(start_frame) file_name = os.path.basename(output) - result = file_name - - # Get the filename pattern match from the output - # path, so we can compute all frames that would - # come out from rendering the ROP node if there - # is a frame pattern in the name - pattern = r"\w+\.(\d+)" + re.escape(ext) - match = re.match(pattern, file_name) - - if match and start_frame is not None: - - # Check if frames are bigger than 1 (file collection) - # override the result - if end_frame - start_frame > 0: - result = self.create_file_list( - match, int(start_frame), int(end_frame) - ) # todo: `frames` currently conflicts with "explicit frames" for a # for a custom frame list. So this should be refactored. + instance.data.update({ - "frames": result, + "frames": file_name, # Set frames to the file name by default. "stagingDir": os.path.dirname(output) }) - @staticmethod - def create_file_list(match, start_frame, end_frame): - """Collect files based on frame range and `regex.match` - - Args: - match(re.match): match object - start_frame(int): start of the animation - end_frame(int): end of the animation - - Returns: - list - - """ - - # Get the padding length - frame = match.group(1) - padding = len(frame) - - # Get the parts of the filename surrounding the frame number, - # so we can put our own frame numbers in. - span = match.span(1) - prefix = match.string[: span[0]] - suffix = match.string[span[1]:] - - # Generate filenames for all frames - result = [] - for i in range(start_frame, end_frame + 1): - - # Format frame number by the padding amount - str_frame = "{number:0{width}d}".format(number=i, width=padding) - - file_name = prefix + str_frame + suffix - result.append(file_name) - - return result + # Skip unnecessary logic if start and end frames are equal. + if start_frame == end_frame: + return + + # Create collection using frame pattern. + # e.g. 'pointcacheBgeoCache_AB010.1001.bgeo' + # will be + frame_collection, _ = clique.assemble( + [file_name], + patterns=[clique.PATTERNS["frames"]], + minimum_items=1 + ) + + # Return as no frame pattern detected. + if not frame_collection: + return + + # It's always expected to be one collection. + frame_collection = frame_collection[0] + frame_collection.indexes.clear() + frame_collection.indexes.update(list(range(start_frame, (end_frame + 1)))) + instance.data["frames"] = list(frame_collection) diff --git a/server_addon/houdini/client/ayon_houdini/version.py b/server_addon/houdini/client/ayon_houdini/version.py index 3dbbb4c23e..4010dbff93 100644 --- a/server_addon/houdini/client/ayon_houdini/version.py +++ b/server_addon/houdini/client/ayon_houdini/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'houdini' version.""" -__version__ = "0.3.7" +__version__ = "0.3.8" diff --git a/server_addon/houdini/package.py b/server_addon/houdini/package.py index c01cc6044d..7e67b169c6 100644 --- a/server_addon/houdini/package.py +++ b/server_addon/houdini/package.py @@ -1,6 +1,6 @@ name = "houdini" title = "Houdini" -version = "0.3.7" +version = "0.3.8" client_dir = "ayon_houdini" diff --git a/server_addon/houdini/server/settings/publish.py b/server_addon/houdini/server/settings/publish.py index 2b88f96922..793f14eae6 100644 --- a/server_addon/houdini/server/settings/publish.py +++ b/server_addon/houdini/server/settings/publish.py @@ -45,6 +45,39 @@ class CollectLocalRenderInstancesModel(BaseSettingsModel): ) +def product_types_enum(): + return [ + {"value": "camera", "label": "Camera (Abc)"}, + {"value": "pointcache", "label": "PointCache (Abc)/PointCache (Bgeo)"}, + {"value": "review", "label": "Review"}, + {"value": "staticMesh", "label": "Static Mesh (FBX)"}, + {"value": "usd", "label": "USD (experimental)"}, + {"value": "vdbcache", "label": "VDB Cache"}, + {"value": "imagesequence", "label": "Composite (Image Sequence)"}, + {"value": "ass", "label": "Arnold ASS"}, + {"value": "arnold_rop", "label": "Arnold ROP"}, + {"value": "mantra_rop", "label": "Mantra ROP"}, + {"value": "redshiftproxy", "label": "Redshift Proxy"}, + {"value": "redshift_rop", "label": "Redshift ROP"}, + {"value": "karma_rop", "label": "Karma ROP"}, + {"value": "vray_rop", "label": "VRay ROP"}, + {"value": "model", "label": "Model"}, + ] + + +class CollectFilesForCleaningUpModel(BaseSettingsModel): + enabled: bool = SettingsField(title="Enabled") + optional: bool = SettingsField(title="Optional") + active: bool = SettingsField(title="Active") + + families: list[str] = SettingsField( + default_factory=list, + enum_resolver=product_types_enum, + conditionalEnum=True, + title="Product Types" + ) + + class ValidateWorkfilePathsModel(BaseSettingsModel): enabled: bool = SettingsField(title="Enabled") optional: bool = SettingsField(title="Optional") @@ -74,6 +107,10 @@ class PublishPluginsModel(BaseSettingsModel): default_factory=CollectChunkSizeModel, title="Collect Chunk Size" ) + CollectFilesForCleaningUp: CollectFilesForCleaningUpModel = SettingsField( + default_factory=CollectFilesForCleaningUpModel, + title="Collect Files For Cleaning Up." + ) CollectLocalRenderInstances: CollectLocalRenderInstancesModel = SettingsField( default_factory=CollectLocalRenderInstancesModel, title="Collect Local Render Instances" @@ -113,6 +150,12 @@ class PublishPluginsModel(BaseSettingsModel): "optional": True, "chunk_size": 999999 }, + "CollectFilesForCleaningUp": { + "enabled": False, + "optional": True, + "active": True, + "families" : [] + }, "CollectLocalRenderInstances": { "use_deadline_aov_filter": False, "aov_filter": {