From 230611e91c6a89f73b8fc9c80a9414107018a0ac Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 3 May 2024 15:58:26 +0200 Subject: [PATCH 01/18] AY-4801 - new creator for editorial_pckg Should publish folder with otio file and (.mov) resources. --- .../create/create_editorial_package.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py new file mode 100644 index 0000000000..6a581b59d1 --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py @@ -0,0 +1,66 @@ +from pathlib import Path + +from ayon_core.pipeline import ( + CreatedInstance, +) + +from ayon_core.lib.attribute_definitions import FileDef +from ayon_core.hosts.traypublisher.api.plugin import TrayPublishCreator + + +class EditorialPackageCreator(TrayPublishCreator): + """Creates instance for OTIO file from published folder. + + Folder contains OTIO file and exported .mov files. Process should publish + whole folder as single `editorial_pckg` product type and (possibly) convert + .mov files into different format and copy them into `publish` `resources` + subfolder. + """ + identifier = "editorial_pckg" + label = "Editorial package" + product_type = "editorial_pckg" + description = "Publish folder with OTIO file and resources" + + # Position batch creator after simple creators + order = 120 + + + def get_icon(self): + return "fa.folder" + + def create(self, product_name, instance_data, pre_create_data): + folder_path = pre_create_data.get("folder_path") + if not folder_path: + return + + instance_data["creator_attributes"] = { + "path": (Path(folder_path["directory"]) / + Path(folder_path["filenames"][0])).as_posix() + } + + # Create new instance + new_instance = CreatedInstance(self.product_type, product_name, + instance_data, self) + self._store_new_instance(new_instance) + + def get_pre_create_attr_defs(self): + # Use same attributes as for instance attributes + return [ + FileDef( + "folder_path", + folders=True, + single_item=True, + extensions=[], + allow_sequences=False, + label="Folder path" + ) + ] + + def get_detail_description(self): + return """# Publish folder with OTIO file and video clips + + Folder contains OTIO file and exported .mov files. Process should + publish whole folder as single `editorial_pckg` product type and + (possibly) convert .mov files into different format and copy them into + `publish` `resources` subfolder. + """ From 4318218881c6d4c0ad0f83bd23395b1c8936f5b7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 3 May 2024 16:02:56 +0200 Subject: [PATCH 02/18] AY-4801 - new collector for editorial_pckg Collects otio_path and resource_paths from folder. Doesn't parse and collect otio_data yet, to not carry too much data over.(Might be changed) --- .../publish/collect_editorial_package.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py new file mode 100644 index 0000000000..101f58b6d1 --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py @@ -0,0 +1,58 @@ +"""Produces instance.data["editorial_pckg"] data used during integration. + +Requires: + instance.data["creator_attributes"]["path"] - from creator + +Provides: + instance -> editorial_pckg (dict): + folder_path (str) + otio_path (str) - from dragged folder + resource_paths (list) + +""" +import os + +import pyblish.api + +from ayon_core.lib.transcoding import VIDEO_EXTENSIONS + + +class CollectEditorialPackage(pyblish.api.InstancePlugin): + """Collects path to OTIO file and resources""" + + label = "Collect Editorial Package" + order = pyblish.api.CollectorOrder - 0.1 + + hosts = ["traypublisher"] + families = ["editorial_pckg"] + + def process(self, instance): + folder_path = instance.data["creator_attributes"].get("path") + if not folder_path or not os.path.exists(folder_path): + self.log.info(( + "Instance doesn't contain collected existing folder path." + )) + return + + instance.data["editorial_pckg"] = {} + instance.data["editorial_pckg"]["folder_path"] = folder_path + + otio_path, resource_paths = ( + self._get_otio_and_resource_paths(folder_path)) + + instance.data["editorial_pckg"]["otio_path"] = otio_path + instance.data["editorial_pckg"]["resource_paths"] = resource_paths + + def _get_otio_and_resource_paths(self, folder_path): + otio_path = None + resource_paths = [] + + file_names = os.listdir(folder_path) + for filename in file_names: + _, ext = os.path.splitext(filename) + file_path = os.path.join(folder_path, filename) + if ext == ".otio": + otio_path = file_path + elif ext in VIDEO_EXTENSIONS: + resource_paths.append(file_path) + return otio_path, resource_paths From 1ff4d63091fbcbdcabc434317181fd19f00a916d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 3 May 2024 16:11:12 +0200 Subject: [PATCH 03/18] AY-4801 - new validator for editorial_pckg Currently checks only by file names and expects flat structure. It ignores path to resources in otio file as folder might be dragged in and published from different location than it was created. --- .../publish/validate_editorial_package.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py new file mode 100644 index 0000000000..869dc73811 --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py @@ -0,0 +1,70 @@ +import os +import opentimelineio + +import pyblish.api +from ayon_core.pipeline import PublishValidationError + + +class ValidateEditorialPackage(pyblish.api.InstancePlugin): + """Checks that published folder contains all resources from otio + + Currently checks only by file names and expects flat structure. + It ignores path to resources in otio file as folder might be dragged in and + published from different location than it was created. + """ + + label = "Validate Editorial Package" + order = pyblish.api.ValidatorOrder - 0.49 + + hosts = ["traypublisher"] + families = ["editorial_pckg"] + + def process(self, instance): + editorial_pckg_data = instance.data.get("editorial_pckg") + if not editorial_pckg_data: + raise PublishValidationError( + f"Editorial package not collected") + + folder_path = editorial_pckg_data["folder_path"] + + otio_path = editorial_pckg_data["otio_path"] + if not otio_path: + raise PublishValidationError( + f"Folder {folder_path} missing otio file") + + resource_paths = editorial_pckg_data["resource_paths"] + + resource_file_names = {os.path.basename(path) + for path in resource_paths} + + otio_data = opentimelineio.adapters.read_from_file(otio_path) + + target_urls = self._get_all_target_urls(otio_data) + missing_files = set() + for target_url in target_urls: + target_basename = os.path.basename(target_url) + if target_basename not in resource_file_names: + missing_files.add(target_basename) + + if missing_files: + raise PublishValidationError("Otio file contains missing files " + f"'{missing_files}'.") + + instance.data["editorial_pckg"]["otio_data"] = otio_data + + def _get_all_target_urls(self, otio_data): + target_urls = [] + + # Iterate through tracks, clips, or other elements + for track in otio_data.tracks: + for clip in track: + # Check if the clip has a media reference + if clip.media_reference is not None: + # Access the target_url from the media reference + target_url = clip.media_reference.target_url + if target_url: + target_urls.append(target_url) + + return target_urls + + From 20dad59947e4fb13d026670baa73820e9e378ecd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 3 May 2024 18:31:40 +0200 Subject: [PATCH 04/18] AY-4801 - added editorial_pckg to integrate --- client/ayon_core/plugins/publish/integrate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index 764168edd3..5a9d8eae2b 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -169,6 +169,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): "yeticacheUE", "tycache", "csv_ingest_file", + "editorial_pckg" ] default_template_name = "publish" From 345f5f31f1a395c4f4d468166bc343933be9974e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 3 May 2024 18:44:48 +0200 Subject: [PATCH 05/18] AY-4801 - added editorial_pckg extractor Modifies otio file with rootless publish paths, prepares for integration. --- .../plugins/publish/extract_editorial_pckg.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py new file mode 100644 index 0000000000..dc8163e1ff --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -0,0 +1,122 @@ +import os.path +import opentimelineio + +import pyblish.api + +from ayon_core.pipeline import publish + + +class ExtractEditorialPackage(publish.Extractor): + """Replaces movie paths in otio file with publish rootless + + Prepares movie resources for integration. + TODO introduce conversion to .mp4 + """ + + label = "Extract Editorial Package" + order = pyblish.api.ExtractorOrder - 0.45 + hosts = ["traypublisher"] + families = ["editorial_pckg"] + + def process(self, instance): + editorial_pckg_data = instance.data.get("editorial_pckg") + + otio_path = editorial_pckg_data["otio_path"] + otio_basename = os.path.basename(otio_path) + staging_dir = self.staging_dir(instance) + + editorial_pckg_repre = { + 'name': "editorial_pckg", + 'ext': "otio", + 'files': otio_basename, + "stagingDir": staging_dir, + } + otio_staging_path = os.path.join(staging_dir, otio_basename) + + instance.data["representations"].append(editorial_pckg_repre) + + publish_path = self._get_published_path(instance) + publish_folder = os.path.dirname(publish_path) + publish_resource_folder = os.path.join(publish_folder, "resources") + + resource_paths = editorial_pckg_data["resource_paths"] + transfers = self._get_transfers(resource_paths, + publish_resource_folder) + if not "transfers" in instance.data: + instance.data["transfers"] = [] + instance.data["transfers"] = transfers + + source_to_rootless = self._get_resource_path_mapping(instance, + transfers) + + otio_data = editorial_pckg_data["otio_data"] + otio_data = self._replace_target_urls(otio_data, source_to_rootless) + + opentimelineio.adapters.write_to_file(otio_data, otio_staging_path) + + self.log.info("Added Editorial Package representation: {}".format( + editorial_pckg_repre)) + + def _get_resource_path_mapping(self, instance, transfers): + """Returns dict of {source_mov_path: rootless_published_path}.""" + replace_paths = {} + anatomy = instance.context.data["anatomy"] + for source, destination in transfers: + rootless_path = self._get_rootless(anatomy, destination) + source_file_name = os.path.basename(source) + replace_paths[source_file_name] = rootless_path + return replace_paths + + def _get_transfers(self, resource_paths, publish_resource_folder): + """Returns list of tuples (source, destination) movie paths.""" + transfers = [] + for res_path in resource_paths: + res_basename = os.path.basename(res_path) + pub_res_path = os.path.join(publish_resource_folder, res_basename) + transfers.append((res_path, pub_res_path)) + return transfers + + def _replace_target_urls(self, otio_data, replace_paths): + """Replace original movie paths with published rootles ones.""" + for track in otio_data.tracks: + for clip in track: + # Check if the clip has a media reference + if clip.media_reference is not None: + # Access the target_url from the media reference + target_url = clip.media_reference.target_url + if not target_url: + continue + file_name = os.path.basename(target_url) + replace_value = replace_paths.get(file_name) + if replace_value: + clip.media_reference.target_url = replace_value + + return otio_data + + def _get_rootless(self, anatomy, path): + """Try to find rootless {root[work]} path from `path`""" + success, rootless_path = anatomy.find_root_template_from_path( + path) + if not success: + # `rootless_path` is not set to `output_dir` if none of roots match + self.log.warning( + f"Could not find root path for remapping '{path}'." + ) + rootless_path = path + + return rootless_path + + def _get_published_path(self, instance): + """Calculates expected `publish` folder""" + # determine published path from Anatomy. + template_data = instance.data.get("anatomyData") + rep = instance.data["representations"][0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None + + anatomy = instance.context.data["anatomy"] + template_data["root"] = anatomy.roots + template = anatomy.get_template_item("publish", "default", "path") + template_filled = template.format_strict(template_data) + return os.path.normpath(template_filled) From 2facf91bcb4a5ea5812dc93b3b2c026978a7a7f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 7 May 2024 13:51:36 +0200 Subject: [PATCH 06/18] AY-4801-Added conversion of resources Added similar configuration as for ExtractReview to control possible conversion from .mov to target format (.mp4) --- .../plugins/publish/extract_editorial_pckg.py | 151 ++++++++++++++++-- .../server/settings/publish_plugins.py | 89 ++++++++++- 2 files changed, 230 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py index dc8163e1ff..02f953d579 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -1,16 +1,20 @@ +import copy import os.path +import subprocess + import opentimelineio import pyblish.api +from ayon_core.lib import filter_profiles, get_ffmpeg_tool_args, run_subprocess from ayon_core.pipeline import publish -class ExtractEditorialPackage(publish.Extractor): +class ExtractEditorialPckgConversion(publish.Extractor): """Replaces movie paths in otio file with publish rootless - Prepares movie resources for integration. - TODO introduce conversion to .mp4 + Prepares movie resources for integration (adds them to `transfers`). + Converts .mov files according to output definition. """ label = "Extract Editorial Package" @@ -35,13 +39,22 @@ def process(self, instance): instance.data["representations"].append(editorial_pckg_repre) - publish_path = self._get_published_path(instance) - publish_folder = os.path.dirname(publish_path) - publish_resource_folder = os.path.join(publish_folder, "resources") - + publish_resource_folder = self._get_publish_resource_folder(instance) resource_paths = editorial_pckg_data["resource_paths"] transfers = self._get_transfers(resource_paths, publish_resource_folder) + + project_settings = instance.context.data["project_settings"] + profiles = (project_settings["traypublisher"] + ["publish"] + ["ExtractEditorialPckgConversion"] + .get("profiles")) + output_def = None + if profiles: + output_def = self._get_output_definition(instance, profiles) + if output_def: + transfers = self._convert_resources(output_def, transfers) + if not "transfers" in instance.data: instance.data["transfers"] = [] instance.data["transfers"] = transfers @@ -57,6 +70,36 @@ def process(self, instance): self.log.info("Added Editorial Package representation: {}".format( editorial_pckg_repre)) + def _get_publish_resource_folder(self, instance): + """Calculates publish folder and create it.""" + publish_path = self._get_published_path(instance) + publish_folder = os.path.dirname(publish_path) + publish_resource_folder = os.path.join(publish_folder, "resources") + + if not os.path.exists(publish_resource_folder): + os.makedirs(publish_resource_folder, exist_ok=True) + return publish_resource_folder + + def _get_output_definition(self, instance, profiles): + """Return appropriate profile by context information.""" + product_type = instance.data["productType"] + product_name = instance.data["productName"] + task_entity = instance.data["taskEntity"] or {} + task_name = task_entity.get("name") + task_type = task_entity.get("taskType") + filtering_criteria = { + "product_types": product_type, + "product_names": product_name, + "task_names": task_name, + "task_types": task_type, + } + profile = filter_profiles( + profiles, + filtering_criteria, + logger=self.log + ) + return profile + def _get_resource_path_mapping(self, instance, transfers): """Returns dict of {source_mov_path: rootless_published_path}.""" replace_paths = {} @@ -68,7 +111,7 @@ def _get_resource_path_mapping(self, instance, transfers): return replace_paths def _get_transfers(self, resource_paths, publish_resource_folder): - """Returns list of tuples (source, destination) movie paths.""" + """Returns list of tuples (source, destination) with movie paths.""" transfers = [] for res_path in resource_paths: res_basename = os.path.basename(res_path) @@ -77,7 +120,7 @@ def _get_transfers(self, resource_paths, publish_resource_folder): return transfers def _replace_target_urls(self, otio_data, replace_paths): - """Replace original movie paths with published rootles ones.""" + """Replace original movie paths with published rootless ones.""" for track in otio_data.tracks: for clip in track: # Check if the clip has a media reference @@ -120,3 +163,93 @@ def _get_published_path(self, instance): template = anatomy.get_template_item("publish", "default", "path") template_filled = template.format_strict(template_data) return os.path.normpath(template_filled) + + def _convert_resources(self, output_def, transfers): + """Converts all resource files to configured format.""" + outputs = output_def["outputs"] + if not outputs: + self.log.warning("No output configured in " + "ayon+settings://traypublisher/publish/ExtractEditorialPckgConversion/profiles/0/outputs") # noqa + return transfers + + final_transfers = [] + # most likely only single output is expected + for output in outputs: + out_extension = output["ext"] + out_def_ffmpeg_args = output["ffmpeg_args"] + ffmpeg_input_args = [ + value.strip() + for value in out_def_ffmpeg_args["input"] + if value.strip() + ] + ffmpeg_video_filters = [ + value.strip() + for value in out_def_ffmpeg_args["video_filters"] + if value.strip() + ] + ffmpeg_audio_filters = [ + value.strip() + for value in out_def_ffmpeg_args["audio_filters"] + if value.strip() + ] + ffmpeg_output_args = [ + value.strip() + for value in out_def_ffmpeg_args["output"] + if value.strip() + ] + ffmpeg_input_args = self._split_ffmpeg_args(ffmpeg_input_args) + + generic_args = [ + subprocess.list2cmdline(get_ffmpeg_tool_args("ffmpeg")) + ] + generic_args.extend(ffmpeg_input_args) + if ffmpeg_video_filters: + generic_args.append("-filter:v") + generic_args.append( + "\"{}\"".format(",".join(ffmpeg_video_filters))) + + if ffmpeg_audio_filters: + generic_args.append("-filter:a") + generic_args.append( + "\"{}\"".format(",".join(ffmpeg_audio_filters))) + + for source, destination in transfers: + base_name = os.path.basename(destination) + file_name, ext = os.path.splitext(base_name) + dest_path = os.path.join(os.path.dirname(destination), + f"{file_name}.{out_extension}") + final_transfers.append((source, dest_path)) + + all_args = copy.deepcopy(generic_args) + all_args.append(f"-i {source}") + all_args.extend(ffmpeg_output_args) # order matters + all_args.append(f"{dest_path}") + subprcs_cmd = " ".join(all_args) + + # run subprocess + self.log.debug("Executing: {}".format(subprcs_cmd)) + run_subprocess(subprcs_cmd, shell=True, logger=self.log) + return final_transfers + + def _split_ffmpeg_args(self, in_args): + """Makes sure all entered arguments are separated in individual items. + + Split each argument string with " -" to identify if string contains + one or more arguments. + """ + splitted_args = [] + for arg in in_args: + sub_args = arg.split(" -") + if len(sub_args) == 1: + if arg and arg not in splitted_args: + splitted_args.append(arg) + continue + + for idx, arg in enumerate(sub_args): + if idx != 0: + arg = "-" + arg + + if arg and arg not in splitted_args: + splitted_args.append(arg) + return splitted_args + diff --git a/server_addon/traypublisher/server/settings/publish_plugins.py b/server_addon/traypublisher/server/settings/publish_plugins.py index f413c86227..9869f54620 100644 --- a/server_addon/traypublisher/server/settings/publish_plugins.py +++ b/server_addon/traypublisher/server/settings/publish_plugins.py @@ -1,4 +1,11 @@ -from ayon_server.settings import BaseSettingsModel, SettingsField +from pydantic import validator + +from ayon_server.settings import ( + BaseSettingsModel, + SettingsField, + task_types_enum, + ensure_unique_names +) class ValidatePluginModel(BaseSettingsModel): @@ -14,6 +21,74 @@ class ValidateFrameRangeModel(ValidatePluginModel): 'my_asset_to_publish.mov')""" +class ExtractEditorialPckgFFmpegModel(BaseSettingsModel): + video_filters: list[str] = SettingsField( + default_factory=list, + title="Video filters" + ) + audio_filters: list[str] = SettingsField( + default_factory=list, + title="Audio filters" + ) + input: list[str] = SettingsField( + default_factory=list, + title="Input arguments" + ) + output: list[str] = SettingsField( + default_factory=list, + title="Output arguments" + ) + + +class ExtractEditorialPckgOutputDefModel(BaseSettingsModel): + """Set extension and ffmpeg arguments. See `ExtractReview` for example.""" + _layout = "expanded" + name: str = SettingsField("", title="Name") + ext: str = SettingsField("", title="Output extension") + + ffmpeg_args: ExtractEditorialPckgFFmpegModel = SettingsField( + default_factory=ExtractEditorialPckgFFmpegModel, + title="FFmpeg arguments" + ) + + +class ExtractEditorialPckgProfileModel(BaseSettingsModel): + product_types: list[str] = SettingsField( + default_factory=list, + title="Product types" + ) + task_types: list[str] = SettingsField( + default_factory=list, + title="Task types", + enum_resolver=task_types_enum + ) + task_names: list[str] = SettingsField( + default_factory=list, + title="Task names" + ) + product_names: list[str] = SettingsField( + default_factory=list, + title="Product names" + ) + outputs: list[ExtractEditorialPckgOutputDefModel] = SettingsField( + default_factory=list, + title="Output Definitions", + ) + + @validator("outputs") + def validate_unique_outputs(cls, value): + ensure_unique_names(value) + return value + + +class ExtractEditorialPckgConversionModel(BaseSettingsModel): + """Conversion of input movie files into expected format.""" + enabled: bool = SettingsField(True) + profiles: list[ExtractEditorialPckgProfileModel] = SettingsField( + default_factory=list, title="Profiles" + ) + + class TrayPublisherPublishPlugins(BaseSettingsModel): CollectFrameDataFromAssetEntity: ValidatePluginModel = SettingsField( default_factory=ValidatePluginModel, @@ -28,6 +103,13 @@ class TrayPublisherPublishPlugins(BaseSettingsModel): default_factory=ValidatePluginModel, ) + ExtractEditorialPckgConversion: ExtractEditorialPckgConversionModel = ( + SettingsField( + default_factory=ExtractEditorialPckgConversionModel, + title="Extract Editorial Package Conversion" + ) + ) + DEFAULT_PUBLISH_PLUGINS = { "CollectFrameDataFromAssetEntity": { @@ -44,5 +126,10 @@ class TrayPublisherPublishPlugins(BaseSettingsModel): "enabled": True, "optional": True, "active": True + }, + "ExtractEditorialPckgConversion": { + "enabled": True, + "optional": True, + "active": True } } From b7be1952e8533a6f794c7604ad4de939060a7495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 9 May 2024 20:24:54 +0200 Subject: [PATCH 07/18] Update client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py Co-authored-by: Roy Nieterau --- .../traypublisher/plugins/create/create_editorial_package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py index 6a581b59d1..19ca032a0f 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py @@ -24,7 +24,6 @@ class EditorialPackageCreator(TrayPublishCreator): # Position batch creator after simple creators order = 120 - def get_icon(self): return "fa.folder" From 2a675f51a6db20322b3e1d5e8f537723bc33154d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:38:16 +0200 Subject: [PATCH 08/18] AY-4801-simplified Settings Got rid of profiles, didn't make much sense. Git rid of multiple output definitions, didn't make sense with single otio file. --- .../server/settings/publish_plugins.py | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/server_addon/traypublisher/server/settings/publish_plugins.py b/server_addon/traypublisher/server/settings/publish_plugins.py index 9869f54620..2afe20c865 100644 --- a/server_addon/traypublisher/server/settings/publish_plugins.py +++ b/server_addon/traypublisher/server/settings/publish_plugins.py @@ -41,9 +41,7 @@ class ExtractEditorialPckgFFmpegModel(BaseSettingsModel): class ExtractEditorialPckgOutputDefModel(BaseSettingsModel): - """Set extension and ffmpeg arguments. See `ExtractReview` for example.""" _layout = "expanded" - name: str = SettingsField("", title="Name") ext: str = SettingsField("", title="Output extension") ffmpeg_args: ExtractEditorialPckgFFmpegModel = SettingsField( @@ -52,40 +50,13 @@ class ExtractEditorialPckgOutputDefModel(BaseSettingsModel): ) -class ExtractEditorialPckgProfileModel(BaseSettingsModel): - product_types: list[str] = SettingsField( - default_factory=list, - title="Product types" - ) - task_types: list[str] = SettingsField( - default_factory=list, - title="Task types", - enum_resolver=task_types_enum - ) - task_names: list[str] = SettingsField( - default_factory=list, - title="Task names" - ) - product_names: list[str] = SettingsField( - default_factory=list, - title="Product names" - ) - outputs: list[ExtractEditorialPckgOutputDefModel] = SettingsField( - default_factory=list, - title="Output Definitions", - ) - - @validator("outputs") - def validate_unique_outputs(cls, value): - ensure_unique_names(value) - return value - - class ExtractEditorialPckgConversionModel(BaseSettingsModel): - """Conversion of input movie files into expected format.""" - enabled: bool = SettingsField(True) - profiles: list[ExtractEditorialPckgProfileModel] = SettingsField( - default_factory=list, title="Profiles" + """Set output definition if resource files should be converted.""" + conversion_enabled: bool = SettingsField(True, + title="Conversion enabled") + output: ExtractEditorialPckgOutputDefModel = SettingsField( + default_factory=ExtractEditorialPckgOutputDefModel, + title="Output Definitions", ) @@ -128,8 +99,7 @@ class TrayPublisherPublishPlugins(BaseSettingsModel): "active": True }, "ExtractEditorialPckgConversion": { - "enabled": True, - "optional": True, + "optional": False, "active": True } } From a1d310fad04e210ac1cf60c86473346c5a3061e4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:39:07 +0200 Subject: [PATCH 09/18] AY-4801-exposed state of conversion from Settings in creator Settings have toggle to on/off conversion, this is exposed for Artist to decide ad-hoc. --- .../create/create_editorial_package.py | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py index 6a581b59d1..72a156dfb7 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_editorial_package.py @@ -4,7 +4,12 @@ CreatedInstance, ) -from ayon_core.lib.attribute_definitions import FileDef +from ayon_core.lib.attribute_definitions import ( + FileDef, + BoolDef, + TextDef, + HiddenDef +) from ayon_core.hosts.traypublisher.api.plugin import TrayPublishCreator @@ -24,6 +29,16 @@ class EditorialPackageCreator(TrayPublishCreator): # Position batch creator after simple creators order = 120 + conversion_enabled = False + + def apply_settings(self, project_settings): + self.conversion_enabled = ( + project_settings["traypublisher"] + ["publish"] + ["ExtractEditorialPckgConversion"] + ["conversion_enabled"] + ) + print(project_settings) def get_icon(self): return "fa.folder" @@ -34,8 +49,9 @@ def create(self, product_name, instance_data, pre_create_data): return instance_data["creator_attributes"] = { - "path": (Path(folder_path["directory"]) / - Path(folder_path["filenames"][0])).as_posix() + "folder_path": (Path(folder_path["directory"]) / + Path(folder_path["filenames"][0])).as_posix(), + "conversion_enabled": pre_create_data["conversion_enabled"] } # Create new instance @@ -53,7 +69,23 @@ def get_pre_create_attr_defs(self): extensions=[], allow_sequences=False, label="Folder path" - ) + ), + BoolDef("conversion_enabled", + tooltip="Convert to output defined in Settings.", + default=self.conversion_enabled, + label="Convert resources"), + ] + + def get_instance_attr_defs(self): + return [ + TextDef( + "folder_path", + label="Folder path", + disabled=True + ), + BoolDef("conversion_enabled", + tooltip="Convert to output defined in Settings.", + label="Convert resources"), ] def get_detail_description(self): From b55ed2e7869a508f33621b656cc03e23e8bd66d2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:39:42 +0200 Subject: [PATCH 10/18] AY-4801-updated variable name Old 'path' clashed in output of instance attributes in Publisher. --- .../traypublisher/plugins/publish/collect_editorial_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py index 101f58b6d1..cb1277546c 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_editorial_package.py @@ -27,7 +27,7 @@ class CollectEditorialPackage(pyblish.api.InstancePlugin): families = ["editorial_pckg"] def process(self, instance): - folder_path = instance.data["creator_attributes"].get("path") + folder_path = instance.data["creator_attributes"]["folder_path"] if not folder_path or not os.path.exists(folder_path): self.log.info(( "Instance doesn't contain collected existing folder path." From 663ace6c8f23ae8faf61408c761aed6457d4c100 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:40:44 +0200 Subject: [PATCH 11/18] AY-4801-conversion controlled by instance attribute --- .../plugins/publish/extract_editorial_pckg.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py index 02f953d579..6b6f0bfe1d 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -45,14 +45,15 @@ def process(self, instance): publish_resource_folder) project_settings = instance.context.data["project_settings"] - profiles = (project_settings["traypublisher"] - ["publish"] - ["ExtractEditorialPckgConversion"] - .get("profiles")) - output_def = None - if profiles: - output_def = self._get_output_definition(instance, profiles) - if output_def: + output_def = (project_settings["traypublisher"] + ["publish"] + ["ExtractEditorialPckgConversion"] + ["output"]) + + conversion_enabled = (instance.data["creator_attributes"] + ["conversion_enabled"]) + + if conversion_enabled and output_def["ext"]: transfers = self._convert_resources(output_def, transfers) if not "transfers" in instance.data: From 0a45c5f8fff5e1bae29cec10033cff3263cfc638 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:41:10 +0200 Subject: [PATCH 12/18] AY-4801-removed profile filtering --- .../plugins/publish/extract_editorial_pckg.py | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py index 6b6f0bfe1d..e2eb4cb916 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -6,7 +6,7 @@ import pyblish.api -from ayon_core.lib import filter_profiles, get_ffmpeg_tool_args, run_subprocess +from ayon_core.lib import get_ffmpeg_tool_args, run_subprocess from ayon_core.pipeline import publish @@ -81,26 +81,6 @@ def _get_publish_resource_folder(self, instance): os.makedirs(publish_resource_folder, exist_ok=True) return publish_resource_folder - def _get_output_definition(self, instance, profiles): - """Return appropriate profile by context information.""" - product_type = instance.data["productType"] - product_name = instance.data["productName"] - task_entity = instance.data["taskEntity"] or {} - task_name = task_entity.get("name") - task_type = task_entity.get("taskType") - filtering_criteria = { - "product_types": product_type, - "product_names": product_name, - "task_names": task_name, - "task_types": task_type, - } - profile = filter_profiles( - profiles, - filtering_criteria, - logger=self.log - ) - return profile - def _get_resource_path_mapping(self, instance, transfers): """Returns dict of {source_mov_path: rootless_published_path}.""" replace_paths = {} From f8503aa5dc7b62e0d5fdd9d982dc55b1204b604a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:41:37 +0200 Subject: [PATCH 13/18] AY-4801-removed multiple output definitions --- .../plugins/publish/extract_editorial_pckg.py | 117 +++++++++--------- 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py index e2eb4cb916..488b8e5a75 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -147,69 +147,66 @@ def _get_published_path(self, instance): def _convert_resources(self, output_def, transfers): """Converts all resource files to configured format.""" - outputs = output_def["outputs"] - if not outputs: - self.log.warning("No output configured in " - "ayon+settings://traypublisher/publish/ExtractEditorialPckgConversion/profiles/0/outputs") # noqa + out_extension = output_def["ext"] + if not out_extension: + self.log.warning("No output extension configured in " + "ayon+settings://traypublisher/publish/ExtractEditorialPckgConversion") # noqa return transfers final_transfers = [] - # most likely only single output is expected - for output in outputs: - out_extension = output["ext"] - out_def_ffmpeg_args = output["ffmpeg_args"] - ffmpeg_input_args = [ - value.strip() - for value in out_def_ffmpeg_args["input"] - if value.strip() - ] - ffmpeg_video_filters = [ - value.strip() - for value in out_def_ffmpeg_args["video_filters"] - if value.strip() - ] - ffmpeg_audio_filters = [ - value.strip() - for value in out_def_ffmpeg_args["audio_filters"] - if value.strip() - ] - ffmpeg_output_args = [ - value.strip() - for value in out_def_ffmpeg_args["output"] - if value.strip() - ] - ffmpeg_input_args = self._split_ffmpeg_args(ffmpeg_input_args) - - generic_args = [ - subprocess.list2cmdline(get_ffmpeg_tool_args("ffmpeg")) - ] - generic_args.extend(ffmpeg_input_args) - if ffmpeg_video_filters: - generic_args.append("-filter:v") - generic_args.append( - "\"{}\"".format(",".join(ffmpeg_video_filters))) - - if ffmpeg_audio_filters: - generic_args.append("-filter:a") - generic_args.append( - "\"{}\"".format(",".join(ffmpeg_audio_filters))) - - for source, destination in transfers: - base_name = os.path.basename(destination) - file_name, ext = os.path.splitext(base_name) - dest_path = os.path.join(os.path.dirname(destination), - f"{file_name}.{out_extension}") - final_transfers.append((source, dest_path)) - - all_args = copy.deepcopy(generic_args) - all_args.append(f"-i {source}") - all_args.extend(ffmpeg_output_args) # order matters - all_args.append(f"{dest_path}") - subprcs_cmd = " ".join(all_args) - - # run subprocess - self.log.debug("Executing: {}".format(subprcs_cmd)) - run_subprocess(subprcs_cmd, shell=True, logger=self.log) + out_def_ffmpeg_args = output_def["ffmpeg_args"] + ffmpeg_input_args = [ + value.strip() + for value in out_def_ffmpeg_args["input"] + if value.strip() + ] + ffmpeg_video_filters = [ + value.strip() + for value in out_def_ffmpeg_args["video_filters"] + if value.strip() + ] + ffmpeg_audio_filters = [ + value.strip() + for value in out_def_ffmpeg_args["audio_filters"] + if value.strip() + ] + ffmpeg_output_args = [ + value.strip() + for value in out_def_ffmpeg_args["output"] + if value.strip() + ] + ffmpeg_input_args = self._split_ffmpeg_args(ffmpeg_input_args) + + generic_args = [ + subprocess.list2cmdline(get_ffmpeg_tool_args("ffmpeg")) + ] + generic_args.extend(ffmpeg_input_args) + if ffmpeg_video_filters: + generic_args.append("-filter:v") + generic_args.append( + "\"{}\"".format(",".join(ffmpeg_video_filters))) + + if ffmpeg_audio_filters: + generic_args.append("-filter:a") + generic_args.append( + "\"{}\"".format(",".join(ffmpeg_audio_filters))) + + for source, destination in transfers: + base_name = os.path.basename(destination) + file_name, ext = os.path.splitext(base_name) + dest_path = os.path.join(os.path.dirname(destination), + f"{file_name}.{out_extension}") + final_transfers.append((source, dest_path)) + + all_args = copy.deepcopy(generic_args) + all_args.append(f"-i {source}") + all_args.extend(ffmpeg_output_args) # order matters + all_args.append(f"{dest_path}") + subprcs_cmd = " ".join(all_args) + + # run subprocess + self.log.debug("Executing: {}".format(subprcs_cmd)) + run_subprocess(subprcs_cmd, shell=True, logger=self.log) return final_transfers def _split_ffmpeg_args(self, in_args): From ba0918964f3766465104d278489d49bad7585233 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 14:41:57 +0200 Subject: [PATCH 14/18] AY-4801-updated validation message --- .../plugins/publish/extract_editorial_pckg.py | 8 +++++--- .../plugins/publish/validate_editorial_package.py | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py index 488b8e5a75..d25a7146a0 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -111,9 +111,11 @@ def _replace_target_urls(self, otio_data, replace_paths): if not target_url: continue file_name = os.path.basename(target_url) - replace_value = replace_paths.get(file_name) - if replace_value: - clip.media_reference.target_url = replace_value + replace_path = replace_paths.get(file_name) + if replace_path: + clip.media_reference.target_url = replace_path + if clip.name == file_name: + clip.name = os.path.basename(replace_path) return otio_data diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py index 869dc73811..ce545610ea 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py @@ -47,8 +47,9 @@ def process(self, instance): missing_files.add(target_basename) if missing_files: - raise PublishValidationError("Otio file contains missing files " - f"'{missing_files}'.") + raise PublishValidationError( + "Otio file contains missing files `{missing_files}`.\n\n" + f"Please add them to `{folder_path}` and republish.") instance.data["editorial_pckg"]["otio_data"] = otio_data From 171ecf2be41c9d6e13e862b7479713d5f3ce221e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 15:53:50 +0200 Subject: [PATCH 15/18] AY-4801-bump up version of OpenTimelineIO --- client/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pyproject.toml b/client/pyproject.toml index 1a0ad7e5f2..5e811321f8 100644 --- a/client/pyproject.toml +++ b/client/pyproject.toml @@ -16,7 +16,7 @@ aiohttp_json_rpc = "*" # TVPaint server aiohttp-middlewares = "^2.0.0" wsrpc_aiohttp = "^3.1.1" # websocket server Click = "^8" -OpenTimelineIO = "0.14.1" +OpenTimelineIO = "0.16.0" opencolorio = "2.2.1" Pillow = "9.5.0" pynput = "^1.7.2" # Timers manager - TODO remove From c976261786de427ba2cff49afee44d5a6e28c99e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 16:04:02 +0200 Subject: [PATCH 16/18] AY-4801-added default setting to .mp4 conversion --- .../server/settings/publish_plugins.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/server_addon/traypublisher/server/settings/publish_plugins.py b/server_addon/traypublisher/server/settings/publish_plugins.py index 2afe20c865..dc659f6110 100644 --- a/server_addon/traypublisher/server/settings/publish_plugins.py +++ b/server_addon/traypublisher/server/settings/publish_plugins.py @@ -100,6 +100,21 @@ class TrayPublisherPublishPlugins(BaseSettingsModel): }, "ExtractEditorialPckgConversion": { "optional": False, - "active": True + "conversion_enabled": True, + "output": { + "ext": "", + "ffmpeg_args": { + "video_filters": [], + "audio_filters": [], + "input": [ + "-apply_trc gamma22" + ], + "output": [ + "-pix_fmt yuv420p", + "-crf 18", + "-intra" + ] + } + } } } From e19986a792972962be07d060249cfe3c56764ca3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 10 May 2024 16:26:10 +0200 Subject: [PATCH 17/18] AY-4801-bump up version of traypublisher package --- server_addon/traypublisher/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server_addon/traypublisher/package.py b/server_addon/traypublisher/package.py index 4ca8ae9fd3..c138a2296d 100644 --- a/server_addon/traypublisher/package.py +++ b/server_addon/traypublisher/package.py @@ -1,3 +1,3 @@ name = "traypublisher" title = "TrayPublisher" -version = "0.1.4" +version = "0.1.5" From 922db60bd031f5a071c5de84b595e9a7a6d5bab2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 10 May 2024 17:18:36 +0200 Subject: [PATCH 18/18] Add quotes to file paths in ExtractEditorialPckgConversion, improve error message handling in ValidateEditorialPackage. --- .../traypublisher/plugins/publish/extract_editorial_pckg.py | 5 ++--- .../plugins/publish/validate_editorial_package.py | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py index d25a7146a0..a4d8d2c6fb 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_editorial_pckg.py @@ -201,9 +201,9 @@ def _convert_resources(self, output_def, transfers): final_transfers.append((source, dest_path)) all_args = copy.deepcopy(generic_args) - all_args.append(f"-i {source}") + all_args.append(f"-i \"{source}\"") all_args.extend(ffmpeg_output_args) # order matters - all_args.append(f"{dest_path}") + all_args.append(f"\"{dest_path}\"") subprcs_cmd = " ".join(all_args) # run subprocess @@ -232,4 +232,3 @@ def _split_ffmpeg_args(self, in_args): if arg and arg not in splitted_args: splitted_args.append(arg) return splitted_args - diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py index ce545610ea..89594ce441 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_editorial_package.py @@ -48,7 +48,7 @@ def process(self, instance): if missing_files: raise PublishValidationError( - "Otio file contains missing files `{missing_files}`.\n\n" + f"Otio file contains missing files `{missing_files}`.\n\n" f"Please add them to `{folder_path}` and republish.") instance.data["editorial_pckg"]["otio_data"] = otio_data @@ -67,5 +67,3 @@ def _get_all_target_urls(self, otio_data): target_urls.append(target_url) return target_urls - -