From 58808104280eba59fed182af0849a66e366450f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 16 Jan 2023 13:44:57 +0100 Subject: [PATCH 01/28] OP-4642 - refactor weird assignment Co-authored-by: Roy Nieterau --- openpype/plugins/publish/extract_color_transcode.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index 62cf8f0deee..835b64b685c 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -138,8 +138,7 @@ def _rename_output_files(self, files_to_convert, output_extension): new_file_name = '{}.{}'.format(file_name, output_extension) renamed_files.append(new_file_name) - files_to_convert = renamed_files - return files_to_convert + return renamed_files def _get_profile(self, instance): """Returns profile if and how repre should be color transcoded.""" From b35ee739bcfcc158daa9ef2d9261f3bda328d4cc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 11:57:51 +0100 Subject: [PATCH 02/28] OP-4643 - added Settings for ExtractColorTranscode --- .../defaults/project_settings/global.json | 4 + .../schemas/schema_global_publish.json | 73 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 0e078dc157b..474e878cb67 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -68,6 +68,10 @@ "output": [] } }, + "ExtractColorTranscode": { + "enabled": true, + "profiles": [] + }, "ExtractReview": { "enabled": true, "profiles": [ diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index 5388d04bc90..46ae6ba5543 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -197,6 +197,79 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "ExtractColorTranscode", + "label": "ExtractColorTranscode", + "checkbox_key": "enabled", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "list", + "key": "profiles", + "label": "Profiles", + "object_type": { + "type": "dict", + "children": [ + { + "key": "families", + "label": "Families", + "type": "list", + "object_type": "text" + }, + { + "key": "hosts", + "label": "Host names", + "type": "hosts-enum", + "multiselection": true + }, + { + "key": "task_types", + "label": "Task types", + "type": "task-types-enum" + }, + { + "key": "task_names", + "label": "Task names", + "type": "list", + "object_type": "text" + }, + { + "key": "subsets", + "label": "Subset names", + "type": "list", + "object_type": "text" + }, + { + "type": "splitter" + }, + { + "key": "ext", + "label": "Output extension", + "type": "text" + }, + { + "key": "output_colorspace", + "label": "Output colorspace", + "type": "text" + }, + { + "key": "custom_tags", + "label": "Custom Tags", + "type": "list", + "object_type": "text" + } + ] + } + } + ] + }, { "type": "dict", "collapsible": true, From 687d6dbf2811db3b8c6f46ee10726b26c1120772 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 11:58:51 +0100 Subject: [PATCH 03/28] OP-4643 - added ExtractColorTranscode Added method to convert from one colorspace to another to transcoding lib --- openpype/lib/transcoding.py | 53 ++++++++ .../publish/extract_color_transcode.py | 124 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 openpype/plugins/publish/extract_color_transcode.py diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index 57279d0380d..6899811ed52 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -1037,3 +1037,56 @@ def convert_ffprobe_fps_to_float(value): if divisor == 0.0: return 0.0 return dividend / divisor + + +def convert_colorspace_for_input_paths( + input_paths, + output_dir, + source_color_space, + target_color_space, + logger=None +): + """Convert source files from one color space to another. + + Filenames of input files are kept so make sure that output directory + is not the same directory as input files have. + - This way it can handle gaps and can keep input filenames without handling + frame template + + Args: + input_paths (str): Paths that should be converted. It is expected that + contains single file or image sequence of samy type. + output_dir (str): Path to directory where output will be rendered. + Must not be same as input's directory. + source_color_space (str): ocio valid color space of source files + target_color_space (str): ocio valid target color space + logger (logging.Logger): Logger used for logging. + + """ + if logger is None: + logger = logging.getLogger(__name__) + + input_arg = "-i" + oiio_cmd = [ + get_oiio_tools_path(), + + # Don't add any additional attributes + "--nosoftwareattrib", + "--colorconvert", source_color_space, target_color_space + ] + for input_path in input_paths: + # Prepare subprocess arguments + + oiio_cmd.extend([ + input_arg, input_path, + ]) + + # Add last argument - path to output + base_filename = os.path.basename(input_path) + output_path = os.path.join(output_dir, base_filename) + oiio_cmd.extend([ + "-o", output_path + ]) + + logger.debug("Conversion command: {}".format(" ".join(oiio_cmd))) + run_subprocess(oiio_cmd, logger=logger) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py new file mode 100644 index 00000000000..58508ab18f9 --- /dev/null +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -0,0 +1,124 @@ +import pyblish.api + +from openpype.pipeline import publish +from openpype.lib import ( + + is_oiio_supported, +) + +from openpype.lib.transcoding import ( + convert_colorspace_for_input_paths, + get_transcode_temp_directory, +) + +from openpype.lib.profiles_filtering import filter_profiles + + +class ExtractColorTranscode(publish.Extractor): + """ + Extractor to convert colors from one colorspace to different. + """ + + label = "Transcode color spaces" + order = pyblish.api.ExtractorOrder + 0.01 + + optional = True + + # Configurable by Settings + profiles = None + options = None + + def process(self, instance): + if not self.profiles: + self.log.warning("No profiles present for create burnin") + return + + if "representations" not in instance.data: + self.log.warning("No representations, skipping.") + return + + if not is_oiio_supported(): + self.log.warning("OIIO not supported, no transcoding possible.") + return + + colorspace_data = instance.data.get("colorspaceData") + if not colorspace_data: + # TODO get_colorspace ?? + self.log.warning("Instance has not colorspace data, skipping") + return + source_color_space = colorspace_data["colorspace"] + + host_name = instance.context.data["hostName"] + family = instance.data["family"] + task_data = instance.data["anatomyData"].get("task", {}) + task_name = task_data.get("name") + task_type = task_data.get("type") + subset = instance.data["subset"] + + filtering_criteria = { + "hosts": host_name, + "families": family, + "task_names": task_name, + "task_types": task_type, + "subset": subset + } + profile = filter_profiles(self.profiles, filtering_criteria, + logger=self.log) + + if not profile: + self.log.info(( + "Skipped instance. None of profiles in presets are for" + " Host: \"{}\" | Families: \"{}\" | Task \"{}\"" + " | Task type \"{}\" | Subset \"{}\" " + ).format(host_name, family, task_name, task_type, subset)) + return + + self.log.debug("profile: {}".format(profile)) + + target_colorspace = profile["output_colorspace"] + if not target_colorspace: + raise RuntimeError("Target colorspace must be set") + + repres = instance.data.get("representations") or [] + for idx, repre in enumerate(repres): + self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) + if not self.repre_is_valid(repre): + continue + + new_staging_dir = get_transcode_temp_directory() + repre["stagingDir"] = new_staging_dir + files_to_remove = repre["files"] + if not isinstance(files_to_remove, list): + files_to_remove = [files_to_remove] + instance.context.data["cleanupFullPaths"].extend(files_to_remove) + + convert_colorspace_for_input_paths( + repre["files"], + new_staging_dir, + source_color_space, + target_colorspace, + self.log + ) + + def repre_is_valid(self, repre): + """Validation if representation should be processed. + + Args: + repre (dict): Representation which should be checked. + + Returns: + bool: False if can't be processed else True. + """ + + if "review" not in (repre.get("tags") or []): + self.log.info(( + "Representation \"{}\" don't have \"review\" tag. Skipped." + ).format(repre["name"])) + return False + + if not repre.get("files"): + self.log.warning(( + "Representation \"{}\" have empty files. Skipped." + ).format(repre["name"])) + return False + return True From e36cf8004706a7d70133d79ea5e0ddd1f208f4c3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 12:05:57 +0100 Subject: [PATCH 04/28] OP-4643 - extractor must run just before ExtractReview Nuke render local is set to 0.01 --- openpype/plugins/publish/extract_color_transcode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index 58508ab18f9..5163cd4045c 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -20,7 +20,7 @@ class ExtractColorTranscode(publish.Extractor): """ label = "Transcode color spaces" - order = pyblish.api.ExtractorOrder + 0.01 + order = pyblish.api.ExtractorOrder + 0.019 optional = True From cb27e5a4d6b512dfa193b031ab266340975245f0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 14:03:22 +0100 Subject: [PATCH 05/28] OP-4643 - fix for full file paths --- .../publish/extract_color_transcode.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index 5163cd4045c..6ad7599f2c2 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -1,3 +1,4 @@ +import os import pyblish.api from openpype.pipeline import publish @@ -41,13 +42,6 @@ def process(self, instance): self.log.warning("OIIO not supported, no transcoding possible.") return - colorspace_data = instance.data.get("colorspaceData") - if not colorspace_data: - # TODO get_colorspace ?? - self.log.warning("Instance has not colorspace data, skipping") - return - source_color_space = colorspace_data["colorspace"] - host_name = instance.context.data["hostName"] family = instance.data["family"] task_data = instance.data["anatomyData"].get("task", {}) @@ -82,18 +76,32 @@ def process(self, instance): repres = instance.data.get("representations") or [] for idx, repre in enumerate(repres): self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) - if not self.repre_is_valid(repre): + # if not self.repre_is_valid(repre): + # continue + + colorspace_data = repre.get("colorspaceData") + if not colorspace_data: + # TODO get_colorspace ?? + self.log.warning("Repre has not colorspace data, skipping") + continue + source_color_space = colorspace_data["colorspace"] + config_path = colorspace_data.get("configData", {}).get("path") + if not os.path.exists(config_path): + self.log.warning("Config file doesn't exist, skipping") continue new_staging_dir = get_transcode_temp_directory() + original_staging_dir = repre["stagingDir"] repre["stagingDir"] = new_staging_dir - files_to_remove = repre["files"] - if not isinstance(files_to_remove, list): - files_to_remove = [files_to_remove] - instance.context.data["cleanupFullPaths"].extend(files_to_remove) + files_to_convert = repre["files"] + if not isinstance(files_to_convert, list): + files_to_convert = [files_to_convert] + files_to_convert = [os.path.join(original_staging_dir, path) + for path in files_to_convert] + instance.context.data["cleanupFullPaths"].extend(files_to_convert) convert_colorspace_for_input_paths( - repre["files"], + files_to_convert, new_staging_dir, source_color_space, target_colorspace, From 7201c57ddc71cac47f6ea38fdbf7d6c1d2d03577 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 14:04:06 +0100 Subject: [PATCH 06/28] OP-4643 - pass path for ocio config --- openpype/lib/transcoding.py | 3 +++ openpype/plugins/publish/extract_color_transcode.py | 1 + 2 files changed, 4 insertions(+) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index 6899811ed52..792e8ddd1e4 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -1042,6 +1042,7 @@ def convert_ffprobe_fps_to_float(value): def convert_colorspace_for_input_paths( input_paths, output_dir, + config_path, source_color_space, target_color_space, logger=None @@ -1058,6 +1059,7 @@ def convert_colorspace_for_input_paths( contains single file or image sequence of samy type. output_dir (str): Path to directory where output will be rendered. Must not be same as input's directory. + config_path (str): path to OCIO config file source_color_space (str): ocio valid color space of source files target_color_space (str): ocio valid target color space logger (logging.Logger): Logger used for logging. @@ -1072,6 +1074,7 @@ def convert_colorspace_for_input_paths( # Don't add any additional attributes "--nosoftwareattrib", + "--colorconfig", config_path, "--colorconvert", source_color_space, target_color_space ] for input_path in input_paths: diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index 6ad7599f2c2..fdb13a47e87 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -103,6 +103,7 @@ def process(self, instance): convert_colorspace_for_input_paths( files_to_convert, new_staging_dir, + config_path, source_color_space, target_colorspace, self.log From dc796b71b4c50d2bb0240667cf9d0f19d85ad5dc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 14:15:33 +0100 Subject: [PATCH 07/28] OP-4643 - add custom_tags --- openpype/plugins/publish/extract_color_transcode.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index fdb13a47e87..ab932b24765 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -72,6 +72,7 @@ def process(self, instance): target_colorspace = profile["output_colorspace"] if not target_colorspace: raise RuntimeError("Target colorspace must be set") + custom_tags = profile["custom_tags"] repres = instance.data.get("representations") or [] for idx, repre in enumerate(repres): @@ -109,6 +110,11 @@ def process(self, instance): self.log ) + if custom_tags: + if not repre.get("custom_tags"): + repre["custom_tags"] = [] + repre["custom_tags"].extend(custom_tags) + def repre_is_valid(self, repre): """Validation if representation should be processed. From 50ff228070729e87dc0a846aa82461fdcc8b08b7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 14:18:38 +0100 Subject: [PATCH 08/28] OP-4643 - added docstring --- openpype/plugins/publish/extract_color_transcode.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index ab932b24765..88e2eed90f8 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -18,6 +18,17 @@ class ExtractColorTranscode(publish.Extractor): """ Extractor to convert colors from one colorspace to different. + + Expects "colorspaceData" on representation. This dictionary is collected + previously and denotes that representation files should be converted. + This dict contains source colorspace information, collected by hosts. + + Target colorspace is selected by profiles in the Settings, based on: + - families + - host + - task types + - task names + - subset names """ label = "Transcode color spaces" From 7a162f9dba79e1c829b9a01338917033607ec074 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 18:15:44 +0100 Subject: [PATCH 09/28] OP-4643 - updated Settings schema --- .../schemas/schema_global_publish.json | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index 46ae6ba5543..c2c911d7d6f 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -247,23 +247,43 @@ "object_type": "text" }, { - "type": "splitter" - }, - { - "key": "ext", - "label": "Output extension", - "type": "text" + "type": "boolean", + "key": "delete_original", + "label": "Delete Original Representation" }, { - "key": "output_colorspace", - "label": "Output colorspace", - "type": "text" + "type": "splitter" }, { - "key": "custom_tags", - "label": "Custom Tags", - "type": "list", - "object_type": "text" + "key": "outputs", + "label": "Output Definitions", + "type": "dict-modifiable", + "highlight_content": true, + "object_type": { + "type": "dict", + "children": [ + { + "key": "output_extension", + "label": "Output extension", + "type": "text" + }, + { + "key": "output_colorspace", + "label": "Output colorspace", + "type": "text" + }, + { + "type": "schema", + "name": "schema_representation_tags" + }, + { + "key": "custom_tags", + "label": "Custom Tags", + "type": "list", + "object_type": "text" + } + ] + } } ] } From 171af695c3ebfdb5a946af8897002e853f34af81 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 18:17:25 +0100 Subject: [PATCH 10/28] OP-4643 - skip video files Only frames currently supported. --- .../plugins/publish/extract_color_transcode.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index 88e2eed90f8..a0714c9a33a 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -36,6 +36,9 @@ class ExtractColorTranscode(publish.Extractor): optional = True + # Supported extensions + supported_exts = ["exr", "jpg", "jpeg", "png", "dpx"] + # Configurable by Settings profiles = None options = None @@ -88,13 +91,7 @@ def process(self, instance): repres = instance.data.get("representations") or [] for idx, repre in enumerate(repres): self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) - # if not self.repre_is_valid(repre): - # continue - - colorspace_data = repre.get("colorspaceData") - if not colorspace_data: - # TODO get_colorspace ?? - self.log.warning("Repre has not colorspace data, skipping") + if not self._repre_is_valid(repre): continue source_color_space = colorspace_data["colorspace"] config_path = colorspace_data.get("configData", {}).get("path") @@ -136,9 +133,9 @@ def repre_is_valid(self, repre): bool: False if can't be processed else True. """ - if "review" not in (repre.get("tags") or []): - self.log.info(( - "Representation \"{}\" don't have \"review\" tag. Skipped." + if repre.get("ext") not in self.supported_exts: + self.log.warning(( + "Representation \"{}\" of unsupported extension. Skipped." ).format(repre["name"])) return False From c7b443519e220ca14dc32ee135c6dcc8085c7d88 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 18:19:08 +0100 Subject: [PATCH 11/28] OP-4643 - refactored profile, delete of original Implemented multiple outputs from single input representation --- .../publish/extract_color_transcode.py | 156 ++++++++++++------ 1 file changed, 109 insertions(+), 47 deletions(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index a0714c9a33a..b0c851d5f40 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -1,4 +1,6 @@ import os +import copy + import pyblish.api from openpype.pipeline import publish @@ -56,13 +58,94 @@ def process(self, instance): self.log.warning("OIIO not supported, no transcoding possible.") return + profile = self._get_profile(instance) + if not profile: + return + + repres = instance.data.get("representations") or [] + for idx, repre in enumerate(list(repres)): + self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) + if not self._repre_is_valid(repre): + continue + + colorspace_data = repre["colorspaceData"] + source_color_space = colorspace_data["colorspace"] + config_path = colorspace_data.get("configData", {}).get("path") + if not os.path.exists(config_path): + self.log.warning("Config file doesn't exist, skipping") + continue + + repre = self._handle_original_repre(repre, profile) + + for _, output_def in profile.get("outputs", {}).items(): + new_repre = copy.deepcopy(repre) + + new_staging_dir = get_transcode_temp_directory() + original_staging_dir = new_repre["stagingDir"] + new_repre["stagingDir"] = new_staging_dir + files_to_convert = new_repre["files"] + if not isinstance(files_to_convert, list): + files_to_convert = [files_to_convert] + + files_to_delete = copy.deepcopy(files_to_convert) + + output_extension = output_def["output_extension"] + files_to_convert = self._rename_output_files(files_to_convert, + output_extension) + + files_to_convert = [os.path.join(original_staging_dir, path) + for path in files_to_convert] + + target_colorspace = output_def["output_colorspace"] + if not target_colorspace: + raise RuntimeError("Target colorspace must be set") + + convert_colorspace_for_input_paths( + files_to_convert, + new_staging_dir, + config_path, + source_color_space, + target_colorspace, + self.log + ) + + instance.context.data["cleanupFullPaths"].extend( + files_to_delete) + + custom_tags = output_def.get("custom_tags") + if custom_tags: + if not new_repre.get("custom_tags"): + new_repre["custom_tags"] = [] + new_repre["custom_tags"].extend(custom_tags) + + # Add additional tags from output definition to representation + for tag in output_def["tags"]: + if tag not in new_repre["tags"]: + new_repre["tags"].append(tag) + + instance.data["representations"].append(new_repre) + + def _rename_output_files(self, files_to_convert, output_extension): + """Change extension of converted files.""" + if output_extension: + output_extension = output_extension.replace('.', '') + renamed_files = [] + for file_name in files_to_convert: + file_name, _ = os.path.splitext(file_name) + new_file_name = '{}.{}'.format(file_name, + output_extension) + renamed_files.append(new_file_name) + files_to_convert = renamed_files + return files_to_convert + + def _get_profile(self, instance): + """Returns profile if and how repre should be color transcoded.""" host_name = instance.context.data["hostName"] family = instance.data["family"] task_data = instance.data["anatomyData"].get("task", {}) task_name = task_data.get("name") task_type = task_data.get("type") subset = instance.data["subset"] - filtering_criteria = { "hosts": host_name, "families": family, @@ -75,55 +158,15 @@ def process(self, instance): if not profile: self.log.info(( - "Skipped instance. None of profiles in presets are for" - " Host: \"{}\" | Families: \"{}\" | Task \"{}\"" - " | Task type \"{}\" | Subset \"{}\" " - ).format(host_name, family, task_name, task_type, subset)) - return + "Skipped instance. None of profiles in presets are for" + " Host: \"{}\" | Families: \"{}\" | Task \"{}\"" + " | Task type \"{}\" | Subset \"{}\" " + ).format(host_name, family, task_name, task_type, subset)) self.log.debug("profile: {}".format(profile)) + return profile - target_colorspace = profile["output_colorspace"] - if not target_colorspace: - raise RuntimeError("Target colorspace must be set") - custom_tags = profile["custom_tags"] - - repres = instance.data.get("representations") or [] - for idx, repre in enumerate(repres): - self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) - if not self._repre_is_valid(repre): - continue - source_color_space = colorspace_data["colorspace"] - config_path = colorspace_data.get("configData", {}).get("path") - if not os.path.exists(config_path): - self.log.warning("Config file doesn't exist, skipping") - continue - - new_staging_dir = get_transcode_temp_directory() - original_staging_dir = repre["stagingDir"] - repre["stagingDir"] = new_staging_dir - files_to_convert = repre["files"] - if not isinstance(files_to_convert, list): - files_to_convert = [files_to_convert] - files_to_convert = [os.path.join(original_staging_dir, path) - for path in files_to_convert] - instance.context.data["cleanupFullPaths"].extend(files_to_convert) - - convert_colorspace_for_input_paths( - files_to_convert, - new_staging_dir, - config_path, - source_color_space, - target_colorspace, - self.log - ) - - if custom_tags: - if not repre.get("custom_tags"): - repre["custom_tags"] = [] - repre["custom_tags"].extend(custom_tags) - - def repre_is_valid(self, repre): + def _repre_is_valid(self, repre): """Validation if representation should be processed. Args: @@ -144,4 +187,23 @@ def repre_is_valid(self, repre): "Representation \"{}\" have empty files. Skipped." ).format(repre["name"])) return False + + if not repre.get("colorspaceData"): + self.log.warning("Repre has not colorspace data, skipping") + return False + return True + + def _handle_original_repre(self, repre, profile): + delete_original = profile["delete_original"] + + if delete_original: + if not repre.get("tags"): + repre["tags"] = [] + + if "review" in repre["tags"]: + repre["tags"].remove("review") + if "delete" not in repre["tags"]: + repre["tags"].append("delete") + + return repre From 69c04bb01dc7bc220318a3d6f63e4ed568bddff2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 18:23:01 +0100 Subject: [PATCH 12/28] OP-4643 - switched logging levels Do not use warning unnecessary. --- openpype/plugins/publish/extract_color_transcode.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index b0c851d5f40..4d38514b8b5 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -47,11 +47,11 @@ class ExtractColorTranscode(publish.Extractor): def process(self, instance): if not self.profiles: - self.log.warning("No profiles present for create burnin") + self.log.debug("No profiles present for color transcode") return if "representations" not in instance.data: - self.log.warning("No representations, skipping.") + self.log.debug("No representations, skipping.") return if not is_oiio_supported(): @@ -177,19 +177,19 @@ def _repre_is_valid(self, repre): """ if repre.get("ext") not in self.supported_exts: - self.log.warning(( + self.log.debug(( "Representation \"{}\" of unsupported extension. Skipped." ).format(repre["name"])) return False if not repre.get("files"): - self.log.warning(( + self.log.debug(( "Representation \"{}\" have empty files. Skipped." ).format(repre["name"])) return False if not repre.get("colorspaceData"): - self.log.warning("Repre has not colorspace data, skipping") + self.log.debug("Repre has no colorspace data. Skipped.") return False return True From 44e12b05b95db1f0b1a43ad506850b5578705942 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 18:46:14 +0100 Subject: [PATCH 13/28] OP-4643 - propagate new extension to representation --- .../publish/extract_color_transcode.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index 4d38514b8b5..62cf8f0deee 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -90,8 +90,13 @@ def process(self, instance): files_to_delete = copy.deepcopy(files_to_convert) output_extension = output_def["output_extension"] - files_to_convert = self._rename_output_files(files_to_convert, - output_extension) + output_extension = output_extension.replace('.', '') + if output_extension: + new_repre["name"] = output_extension + new_repre["ext"] = output_extension + + files_to_convert = self._rename_output_files( + files_to_convert, output_extension) files_to_convert = [os.path.join(original_staging_dir, path) for path in files_to_convert] @@ -127,15 +132,13 @@ def process(self, instance): def _rename_output_files(self, files_to_convert, output_extension): """Change extension of converted files.""" - if output_extension: - output_extension = output_extension.replace('.', '') - renamed_files = [] - for file_name in files_to_convert: - file_name, _ = os.path.splitext(file_name) - new_file_name = '{}.{}'.format(file_name, - output_extension) - renamed_files.append(new_file_name) - files_to_convert = renamed_files + renamed_files = [] + for file_name in files_to_convert: + file_name, _ = os.path.splitext(file_name) + new_file_name = '{}.{}'.format(file_name, + output_extension) + renamed_files.append(new_file_name) + files_to_convert = renamed_files return files_to_convert def _get_profile(self, instance): From 85fc41bd7427a395fbf1454b12a6ee6fea34e594 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Jan 2023 18:46:35 +0100 Subject: [PATCH 14/28] OP-4643 - added label to Settings --- .../projects_schema/schemas/schema_global_publish.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index c2c911d7d6f..7155510fefd 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -201,10 +201,14 @@ "type": "dict", "collapsible": true, "key": "ExtractColorTranscode", - "label": "ExtractColorTranscode", + "label": "ExtractColorTranscode (ImageIO)", "checkbox_key": "enabled", "is_group": true, "children": [ + { + "type": "label", + "label": "Configure output format(s) and color spaces for matching representations. Empty 'Output extension' denotes keeping source extension." + }, { "type": "boolean", "key": "enabled", From 53470bb0333cd8662e3bf4433fa78898bb29eb19 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 18 Jan 2023 09:56:29 +0000 Subject: [PATCH 15/28] Initial draft --- openpype/hosts/maya/api/lib_renderproducts.py | 73 ++++++++++++++----- .../maya/plugins/publish/collect_render.py | 4 +- openpype/hosts/maya/startup/userSetup.py | 8 +- .../plugins/publish/submit_publish_job.py | 33 ++++++++- 4 files changed, 92 insertions(+), 26 deletions(-) diff --git a/openpype/hosts/maya/api/lib_renderproducts.py b/openpype/hosts/maya/api/lib_renderproducts.py index c54e3ab3e0c..b76b4415883 100644 --- a/openpype/hosts/maya/api/lib_renderproducts.py +++ b/openpype/hosts/maya/api/lib_renderproducts.py @@ -127,6 +127,7 @@ class RenderProduct(object): """ productName = attr.ib() ext = attr.ib() # extension + colorspace = attr.ib() # colorspace aov = attr.ib(default=None) # source aov driver = attr.ib(default=None) # source driver multipart = attr.ib(default=False) # multichannel file @@ -344,7 +345,6 @@ def match_last(tokens, text): separator = file_prefix[matches[0].end(1):matches[1].start(1)] return separator - def _get_layer_data(self): # type: () -> LayerMetadata # ______________________________________________ @@ -553,6 +553,9 @@ def _get_aov_render_products(self, aov, cameras=None): ] for ai_driver in ai_drivers: + colorspace = self._get_colorspace( + ai_driver + ".colorManagement" + ) # todo: check aiAOVDriver.prefix as it could have # a custom path prefix set for this driver @@ -590,12 +593,15 @@ def _get_aov_render_products(self, aov, cameras=None): global_aov = self._get_attr(aov, "globalAov") if global_aov: for camera in cameras: - product = RenderProduct(productName=name, - ext=ext, - aov=aov_name, - driver=ai_driver, - multipart=multipart, - camera=camera) + product = RenderProduct( + productName=name, + ext=ext, + aov=aov_name, + driver=ai_driver, + multipart=multipart, + camera=camera, + colorspace=colorspace + ) products.append(product) all_light_groups = self._get_attr(aov, "lightGroups") @@ -603,13 +609,16 @@ def _get_aov_render_products(self, aov, cameras=None): # All light groups is enabled. A single multipart # Render Product for camera in cameras: - product = RenderProduct(productName=name + "_lgroups", - ext=ext, - aov=aov_name, - driver=ai_driver, - # Always multichannel output - multipart=True, - camera=camera) + product = RenderProduct( + productName=name + "_lgroups", + ext=ext, + aov=aov_name, + driver=ai_driver, + # Always multichannel output + multipart=True, + camera=camera, + colorspace=colorspace + ) products.append(product) else: value = self._get_attr(aov, "lightGroupsList") @@ -625,12 +634,28 @@ def _get_aov_render_products(self, aov, cameras=None): aov=aov_name, driver=ai_driver, ext=ext, - camera=camera + camera=camera, + colorspace=colorspace ) products.append(product) return products + def _get_colorspace(self, attribute): + """Resolve colorspace from Arnold settings.""" + + def _view_transform(): + preferences = lib.get_color_management_preferences() + return preferences["view_transform"] + + resolved_values = { + "Raw": lambda: "Raw", + "Use View Transform": _view_transform, + # Default. Same as Maya Preferences. + "Use Output Transform": lib.get_color_management_output_transform + } + return resolved_values[self._get_attr(attribute)]() + def get_render_products(self): """Get all AOVs. @@ -659,11 +684,19 @@ def get_render_products(self): ] default_ext = self._get_attr("defaultRenderGlobals.imfPluginKey") - beauty_products = [RenderProduct( - productName="beauty", - ext=default_ext, - driver="defaultArnoldDriver", - camera=camera) for camera in cameras] + colorspace = self._get_colorspace( + "defaultArnoldDriver.colorManagement" + ) + beauty_products = [ + RenderProduct( + productName="beauty", + ext=default_ext, + driver="defaultArnoldDriver", + camera=camera, + colorspace=colorspace + ) for camera in cameras + ] + # AOVs > Legacy > Maya Render View > Mode aovs_enabled = bool( self._get_attr("defaultArnoldRenderOptions.aovMode") diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index b1ad3ca58ed..2c894243815 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -42,7 +42,6 @@ import re import os import platform -import json from maya import cmds import maya.app.renderSetup.model.renderSetup as renderSetup @@ -318,6 +317,9 @@ def process(self, context): "aovSeparator": layer_render_products.layer_data.aov_separator, # noqa: E501 "renderSetupIncludeLights": render_instance.data.get( "renderSetupIncludeLights" + ), + "colorspaceConfig": ( + lib.get_color_management_preferences()["config"] ) } diff --git a/openpype/hosts/maya/startup/userSetup.py b/openpype/hosts/maya/startup/userSetup.py index 40cd51f2d82..cb5aa4a8980 100644 --- a/openpype/hosts/maya/startup/userSetup.py +++ b/openpype/hosts/maya/startup/userSetup.py @@ -4,10 +4,16 @@ from openpype.hosts.maya.api import MayaHost from maya import cmds +# MAYA_RESOURCES enviornment variable is referenced in default OCIO path but +# it's not part of the environment. Patching this so it works as expected. +if "MAYA_RESOURCES" not in os.environ: + os.environ["MAYA_RESOURCES"] = os.path.join( + os.environ["MAYA_LOCATION"], "resources" + ).replace("\\", "/") + host = MayaHost() install_host(host) - print("starting OpenPype usersetup") # build a shelf diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 7e39a644a29..8811fa5d344 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -427,7 +427,9 @@ def _copy_extend_frames(self, instance, representation): self.log.info( "Finished copying %i files" % len(resource_files)) - def _create_instances_for_aov(self, instance_data, exp_files): + def _create_instances_for_aov( + self, instance_data, exp_files, additional_data + ): """Create instance for each AOV found. This will create new instance for every aov it can detect in expected @@ -528,6 +530,14 @@ def _create_instances_for_aov(self, instance_data, exp_files): else: files = os.path.basename(col) + # Copy render product "colorspace" data to representation. + colorspace = "" + products = additional_data["renderProducts"].layer_data.products + for product in products: + if product.productName == aov: + colorspace = product.colorspace + break + rep = { "name": ext, "ext": ext, @@ -537,7 +547,14 @@ def _create_instances_for_aov(self, instance_data, exp_files): # If expectedFile are absolute, we need only filenames "stagingDir": staging, "fps": new_instance.get("fps"), - "tags": ["review"] if preview else [] + "tags": ["review"] if preview else [], + "colorspaceData": { + "colorspace": colorspace, + "configData": { + "path": additional_data["colorspaceConfig"], + "template": "" + } + } } # support conversion from tiled to scanline @@ -561,7 +578,7 @@ def _create_instances_for_aov(self, instance_data, exp_files): self.log.debug("instances:{}".format(instances)) return instances - def _get_representations(self, instance, exp_files): + def _get_representations(self, instance, exp_files, additional_data): """Create representations for file sequences. This will return representations of expected files if they are not @@ -897,6 +914,11 @@ def process(self, instance): self.log.info(data.get("expectedFiles")) + additional_data = { + "renderProducts": instance.data["renderProducts"], + "colorspaceConfig": instance.data["colorspaceConfig"] + } + if isinstance(data.get("expectedFiles")[0], dict): # we cannot attach AOVs to other subsets as we consider every # AOV subset of its own. @@ -911,12 +933,15 @@ def process(self, instance): # there are multiple renderable cameras in scene) instances = self._create_instances_for_aov( instance_skeleton_data, - data.get("expectedFiles")) + data.get("expectedFiles"), + additional_data + ) self.log.info("got {} instance{}".format( len(instances), "s" if len(instances) > 1 else "")) else: + #Need to inject colorspace here. representations = self._get_representations( instance_skeleton_data, data.get("expectedFiles") From ab9b2d5970b458447e442d9d031bc3d8e76ddc7e Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 18 Jan 2023 15:13:12 +0000 Subject: [PATCH 16/28] Missing initial commit --- openpype/hosts/maya/api/lib.py | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index dd5da275e89..95b4db1b42b 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -5,6 +5,7 @@ import platform import uuid import math +import re import json import logging @@ -3446,3 +3447,47 @@ def dgcontext(mtime): def get_attribute_input(attr): connections = cmds.listConnections(attr, plugs=True, destination=False) return connections[0] if connections else None + + +def get_color_management_preferences(): + """Get and resolve OCIO preferences.""" + data = { + # Is color management enabled. + "enabled": cmds.colorManagementPrefs( + query=True, cmEnabled=True + ), + "rendering_space": cmds.colorManagementPrefs( + query=True, renderingSpaceName=True + ), + "output_transform": cmds.colorManagementPrefs( + query=True, outputTransformName=True + ), + "output_transform_enabled": cmds.colorManagementPrefs( + query=True, outputTransformEnabled=True + ), + "view_transform": cmds.colorManagementPrefs( + query=True, viewTransformName=True + ) + } + + path = cmds.colorManagementPrefs( + query=True, configFilePath=True + ) + # Resolve environment variables in config path. "MAYA_RESOURCES" are in the + # path by default. + for group in re.search(r'(<.*>)', path).groups(): + path = path.replace( + group, os.environ[group[1:-1]] + ) + + data["config"] = path + + return data + + +def get_color_management_output_transform(): + preferences = get_color_management_preferences() + colorspace = preferences["rendering_space"] + if preferences["output_transform_enabled"]: + colorspace = preferences["output_transform"] + return colorspace From 3279634dacd18e6d38a29708596c822d276781c6 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 18 Jan 2023 17:57:28 +0000 Subject: [PATCH 17/28] Fix Environment variable substitution in path --- openpype/hosts/maya/api/lib.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index 95b4db1b42b..b4aa18af654 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -3473,12 +3473,14 @@ def get_color_management_preferences(): path = cmds.colorManagementPrefs( query=True, configFilePath=True ) + # Resolve environment variables in config path. "MAYA_RESOURCES" are in the # path by default. - for group in re.search(r'(<.*>)', path).groups(): - path = path.replace( - group, os.environ[group[1:-1]] - ) + def _subst_with_env_value(match): + key = match.group(1) + return os.environ.get(key, "") + + path = re.sub(r'<([^>]+)>', _subst_with_env_value, path) data["config"] = path From 7a7f87e861abf4aae7fb3a96f660d74fe4866329 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 19 Jan 2023 09:30:11 +0000 Subject: [PATCH 18/28] Improvement Cleaner way to resolve in path. --- openpype/hosts/maya/api/lib.py | 11 ++++------- openpype/hosts/maya/startup/userSetup.py | 6 ------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index b4aa18af654..0807db88dc8 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -3474,13 +3474,10 @@ def get_color_management_preferences(): query=True, configFilePath=True ) - # Resolve environment variables in config path. "MAYA_RESOURCES" are in the - # path by default. - def _subst_with_env_value(match): - key = match.group(1) - return os.environ.get(key, "") - - path = re.sub(r'<([^>]+)>', _subst_with_env_value, path) + # The OCIO config supports a custom token. + maya_resources_token = "" + maya_resources_path = om.MGlobal.getAbsolutePathToResources() + path = path.replace(maya_resources_token, maya_resources_path) data["config"] = path diff --git a/openpype/hosts/maya/startup/userSetup.py b/openpype/hosts/maya/startup/userSetup.py index cb5aa4a8980..1104421cd9d 100644 --- a/openpype/hosts/maya/startup/userSetup.py +++ b/openpype/hosts/maya/startup/userSetup.py @@ -4,12 +4,6 @@ from openpype.hosts.maya.api import MayaHost from maya import cmds -# MAYA_RESOURCES enviornment variable is referenced in default OCIO path but -# it's not part of the environment. Patching this so it works as expected. -if "MAYA_RESOURCES" not in os.environ: - os.environ["MAYA_RESOURCES"] = os.path.join( - os.environ["MAYA_LOCATION"], "resources" - ).replace("\\", "/") host = MayaHost() install_host(host) From c2d0bc8f39f69fc8f2cea36972242e946a6c9deb Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 19 Jan 2023 09:34:55 +0000 Subject: [PATCH 19/28] HOund --- openpype/hosts/maya/api/lib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index 0807db88dc8..23f7319d4ab 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -5,7 +5,6 @@ import platform import uuid import math -import re import json import logging From 3fa7610061298e4a3de96d176b74f5bd3ecca652 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 31 Jan 2023 18:35:50 +0000 Subject: [PATCH 20/28] Working version for Arnold. --- openpype/hosts/maya/api/lib.py | 13 +++++++++++-- openpype/hosts/maya/api/lib_renderproducts.py | 6 +++++- .../hosts/maya/plugins/publish/collect_render.py | 8 ++++---- .../deadline/plugins/publish/submit_publish_job.py | 10 +++++++--- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index f869dadaadb..b31ab2408b7 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -14,7 +14,7 @@ from six import string_types from maya import cmds, mel -import maya.api.OpenMaya as om +from maya.api import OpenMaya from openpype.client import ( get_project, @@ -3402,13 +3402,22 @@ def get_color_management_preferences(): ) } + # Split view and display from view_transform. view_transform comes in + # format of "{view} ({display})". + display = data["view_transform"].split("(")[-1].replace(")", "") + data.update({ + "display": display, + "view": data["view_transform"].replace("({})".format(display), "")[:-1] + }) + + # Get config absolute path. path = cmds.colorManagementPrefs( query=True, configFilePath=True ) # The OCIO config supports a custom token. maya_resources_token = "" - maya_resources_path = om.MGlobal.getAbsolutePathToResources() + maya_resources_path = OpenMaya.MGlobal.getAbsolutePathToResources() path = path.replace(maya_resources_token, maya_resources_path) data["config"] = path diff --git a/openpype/hosts/maya/api/lib_renderproducts.py b/openpype/hosts/maya/api/lib_renderproducts.py index 0b585dc8cb2..58ccbfd5a21 100644 --- a/openpype/hosts/maya/api/lib_renderproducts.py +++ b/openpype/hosts/maya/api/lib_renderproducts.py @@ -648,8 +648,12 @@ def _view_transform(): preferences = lib.get_color_management_preferences() return preferences["view_transform"] + def _raw(): + preferences = lib.get_color_management_preferences() + return preferences["rendering_space"] + resolved_values = { - "Raw": lambda: "Raw", + "Raw": _raw, "Use View Transform": _view_transform, # Default. Same as Maya Preferences. "Use Output Transform": lib.get_color_management_output_transform diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index 2c894243815..d0164f30a8b 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -264,7 +264,7 @@ def process(self, context): self.log.info(full_exp_files) self.log.info("collecting layer: {}".format(layer_name)) # Get layer specific settings, might be overrides - + colorspace_data = lib.get_color_management_preferences() data = { "subset": expected_layer_name, "attachTo": attach_to, @@ -318,9 +318,9 @@ def process(self, context): "renderSetupIncludeLights": render_instance.data.get( "renderSetupIncludeLights" ), - "colorspaceConfig": ( - lib.get_color_management_preferences()["config"] - ) + "colorspaceConfig": colorspace_data["config"], + "colorspaceDisplay": colorspace_data["display"], + "colorspaceView": colorspace_data["view"] } # Collect Deadline url if Deadline module is enabled diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 8811fa5d344..02aa1043d18 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -550,10 +550,12 @@ def _create_instances_for_aov( "tags": ["review"] if preview else [], "colorspaceData": { "colorspace": colorspace, - "configData": { + "config": { "path": additional_data["colorspaceConfig"], "template": "" - } + }, + "display": additional_data["display"], + "view": additional_data["view"] } } @@ -916,7 +918,9 @@ def process(self, instance): additional_data = { "renderProducts": instance.data["renderProducts"], - "colorspaceConfig": instance.data["colorspaceConfig"] + "colorspaceConfig": instance.data["colorspaceConfig"], + "display": instance.data["colorspaceDisplay"], + "view": instance.data["colorspaceView"] } if isinstance(data.get("expectedFiles")[0], dict): From 5c4856d72a302db1132b21f8fac87e751e045899 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 1 Feb 2023 12:05:38 +0000 Subject: [PATCH 21/28] BigRoy feedback --- openpype/hosts/maya/api/lib.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index b31ab2408b7..d83f18bf30d 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -5,6 +5,7 @@ import platform import uuid import math +import re import json import logging @@ -3404,10 +3405,11 @@ def get_color_management_preferences(): # Split view and display from view_transform. view_transform comes in # format of "{view} ({display})". - display = data["view_transform"].split("(")[-1].replace(")", "") + regex = re.compile(r"^(?P.+) \((?P.+)\)$") + match = regex.match(data["view_transform"]) data.update({ - "display": display, - "view": data["view_transform"].replace("({})".format(display), "")[:-1] + "display": match.group("display"), + "view": match.group("view") }) # Get config absolute path. From 89d7edd8dfe0e6917c18870c92628659a8635541 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 1 Feb 2023 12:06:26 +0000 Subject: [PATCH 22/28] Support for Maya Hardware --- openpype/hosts/maya/api/lib_renderproducts.py | 7 ++++++- openpype/hosts/maya/api/lib_rendersettings.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/api/lib_renderproducts.py b/openpype/hosts/maya/api/lib_renderproducts.py index 58ccbfd5a21..6a607e7ff3a 100644 --- a/openpype/hosts/maya/api/lib_renderproducts.py +++ b/openpype/hosts/maya/api/lib_renderproducts.py @@ -1354,7 +1354,12 @@ def get_render_products(self): products = [] for cam in self.get_renderable_cameras(): - product = RenderProduct(productName="beauty", ext=ext, camera=cam) + product = RenderProduct( + productName="beauty", + ext=ext, + camera=cam, + colorspace=lib.get_color_management_output_transform() + ) products.append(product) return products diff --git a/openpype/hosts/maya/api/lib_rendersettings.py b/openpype/hosts/maya/api/lib_rendersettings.py index 6190a494018..4710b101286 100644 --- a/openpype/hosts/maya/api/lib_rendersettings.py +++ b/openpype/hosts/maya/api/lib_rendersettings.py @@ -23,7 +23,8 @@ class RenderSettings(object): 'vray': 'vraySettings.fileNamePrefix', 'arnold': 'defaultRenderGlobals.imageFilePrefix', 'renderman': 'rmanGlobals.imageFileFormat', - 'redshift': 'defaultRenderGlobals.imageFilePrefix' + 'redshift': 'defaultRenderGlobals.imageFilePrefix', + 'mayahardware2': 'defaultRenderGlobals.imageFilePrefix' } _image_prefixes = { From 21d275d46ae2e7d8c1cc9fae67cd8512da46dec1 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 1 Feb 2023 12:27:48 +0000 Subject: [PATCH 23/28] Support for Vray --- openpype/hosts/maya/api/lib_renderproducts.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/openpype/hosts/maya/api/lib_renderproducts.py b/openpype/hosts/maya/api/lib_renderproducts.py index 6a607e7ff3a..38415c2ae25 100644 --- a/openpype/hosts/maya/api/lib_renderproducts.py +++ b/openpype/hosts/maya/api/lib_renderproducts.py @@ -841,9 +841,13 @@ def get_render_products(self): if not dont_save_rgb: for camera in cameras: products.append( - RenderProduct(productName="", - ext=default_ext, - camera=camera)) + RenderProduct( + productName="", + ext=default_ext, + camera=camera, + colorspace=lib.get_color_management_output_transform() + ) + ) # separate alpha file separate_alpha = self._get_attr("vraySettings.separateAlpha") @@ -895,10 +899,13 @@ def get_render_products(self): aov_name = self._get_vray_aov_name(aov) for camera in cameras: - product = RenderProduct(productName=aov_name, - ext=default_ext, - aov=aov, - camera=camera) + product = RenderProduct( + productName=aov_name, + ext=default_ext, + aov=aov, + camera=camera, + colorspace=lib.get_color_management_output_transform() + ) products.append(product) return products From d39dd9ce1de40544cbdf612c6f9e5ff00b126129 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 13 Feb 2023 17:59:40 +0000 Subject: [PATCH 24/28] Correct colorspace when using "View Transform" --- openpype/hosts/maya/api/lib_renderproducts.py | 7 +- openpype/pipeline/colorspace.py | 99 +++++++++---------- openpype/scripts/ocio_wrapper.py | 20 +++- 3 files changed, 70 insertions(+), 56 deletions(-) diff --git a/openpype/hosts/maya/api/lib_renderproducts.py b/openpype/hosts/maya/api/lib_renderproducts.py index 38415c2ae25..4930143fb15 100644 --- a/openpype/hosts/maya/api/lib_renderproducts.py +++ b/openpype/hosts/maya/api/lib_renderproducts.py @@ -46,6 +46,7 @@ from . import lib from . import lib_rendersetup +from openpype.pipeline.colorspace import get_ocio_config_views from maya import cmds, mel @@ -646,7 +647,11 @@ def _get_colorspace(self, attribute): def _view_transform(): preferences = lib.get_color_management_preferences() - return preferences["view_transform"] + views_data = get_ocio_config_views(preferences["config"]) + view_data = views_data[ + "{}/{}".format(preferences["display"], preferences["view"]) + ] + return view_data["colorspace"] def _raw(): preferences = lib.get_color_management_preferences() diff --git a/openpype/pipeline/colorspace.py b/openpype/pipeline/colorspace.py index cb37b2c4ae5..6f68bdc5bf6 100644 --- a/openpype/pipeline/colorspace.py +++ b/openpype/pipeline/colorspace.py @@ -198,41 +198,19 @@ def validate_imageio_colorspace_in_config(config_path, colorspace_name): return True -def get_ocio_config_colorspaces(config_path): - """Get all colorspace data - - Wrapper function for aggregating all names and its families. - Families can be used for building menu and submenus in gui. - - Args: - config_path (str): path leading to config.ocio file - - Returns: - dict: colorspace and family in couple - """ - if sys.version_info[0] == 2: - return get_colorspace_data_subprocess(config_path) - - from ..scripts.ocio_wrapper import _get_colorspace_data - return _get_colorspace_data(config_path) - - -def get_colorspace_data_subprocess(config_path): - """Get colorspace data via subprocess +def get_data_subprocess(config_path, data_type): + """Get data via subprocess Wrapper for Python 2 hosts. Args: config_path (str): path leading to config.ocio file - - Returns: - dict: colorspace and family in couple """ with _make_temp_json_file() as tmp_json_path: # Prepare subprocess arguments args = [ "run", get_ocio_config_script_path(), - "config", "get_colorspace", + "config", data_type, "--in_path", config_path, "--out_path", tmp_json_path @@ -251,6 +229,47 @@ def get_colorspace_data_subprocess(config_path): return json.loads(return_json_data) +def compatible_python(): + """Only 3.9 or higher can directly use PyOpenColorIO in ocio_wrapper""" + compatible = False + if sys.version[0] == 3 and sys.version[1] >= 9: + compatible = True + return compatible + + +def get_ocio_config_colorspaces(config_path): + """Get all colorspace data + + Wrapper function for aggregating all names and its families. + Families can be used for building menu and submenus in gui. + + Args: + config_path (str): path leading to config.ocio file + + Returns: + dict: colorspace and family in couple + """ + if compatible_python(): + from ..scripts.ocio_wrapper import _get_colorspace_data + return _get_colorspace_data(config_path) + else: + return get_colorspace_data_subprocess(config_path) + + +def get_colorspace_data_subprocess(config_path): + """Get colorspace data via subprocess + + Wrapper for Python 2 hosts. + + Args: + config_path (str): path leading to config.ocio file + + Returns: + dict: colorspace and family in couple + """ + return get_data_subprocess(config_path, "get_colorspace") + + def get_ocio_config_views(config_path): """Get all viewer data @@ -263,12 +282,12 @@ def get_ocio_config_views(config_path): Returns: dict: `display/viewer` and viewer data """ - if sys.version_info[0] == 2: + if compatible_python(): + from ..scripts.ocio_wrapper import _get_views_data + return _get_views_data(config_path) + else: return get_views_data_subprocess(config_path) - from ..scripts.ocio_wrapper import _get_views_data - return _get_views_data(config_path) - def get_views_data_subprocess(config_path): """Get viewers data via subprocess @@ -281,27 +300,7 @@ def get_views_data_subprocess(config_path): Returns: dict: `display/viewer` and viewer data """ - with _make_temp_json_file() as tmp_json_path: - # Prepare subprocess arguments - args = [ - "run", get_ocio_config_script_path(), - "config", "get_views", - "--in_path", config_path, - "--out_path", tmp_json_path - - ] - log.info("Executing: {}".format(" ".join(args))) - - process_kwargs = { - "logger": log, - "env": {} - } - - run_openpype_process(*args, **process_kwargs) - - # return all colorspaces - return_json_data = open(tmp_json_path).read() - return json.loads(return_json_data) + return get_data_subprocess(config_path, "get_views") def get_imageio_config( diff --git a/openpype/scripts/ocio_wrapper.py b/openpype/scripts/ocio_wrapper.py index 0685b2e52a1..aa9e11c8411 100644 --- a/openpype/scripts/ocio_wrapper.py +++ b/openpype/scripts/ocio_wrapper.py @@ -157,11 +157,21 @@ def _get_views_data(config_path): config = ocio.Config().CreateFromFile(str(config_path)) - return { - f"{d}/{v}": {"display": d, "view": v} - for d in config.getDisplays() - for v in config.getViews(d) - } + data = {} + for display in config.getDisplays(): + for view in config.getViews(display): + colorspace = config.getDisplayViewColorSpaceName(display, view) + # Special token. See https://opencolorio.readthedocs.io/en/latest/guides/authoring/authoring.html#shared-views + if colorspace == "": + colorspace = display + + data[f"{display}/{view}"] = { + "display": display, + "view": view, + "colorspace": colorspace + } + + return data if __name__ == '__main__': From e21323c78841902bf66f8670b1890a7cde11773c Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 13 Feb 2023 18:01:38 +0000 Subject: [PATCH 25/28] Hound --- openpype/scripts/ocio_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/scripts/ocio_wrapper.py b/openpype/scripts/ocio_wrapper.py index aa9e11c8411..16558642c63 100644 --- a/openpype/scripts/ocio_wrapper.py +++ b/openpype/scripts/ocio_wrapper.py @@ -161,7 +161,7 @@ def _get_views_data(config_path): for display in config.getDisplays(): for view in config.getViews(display): colorspace = config.getDisplayViewColorSpaceName(display, view) - # Special token. See https://opencolorio.readthedocs.io/en/latest/guides/authoring/authoring.html#shared-views + # Special token. See https://opencolorio.readthedocs.io/en/latest/guides/authoring/authoring.html#shared-views # noqa if colorspace == "": colorspace = display From 2621016015b0b2c3e36f36d4c3853851d5154256 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 14 Feb 2023 16:27:43 +0000 Subject: [PATCH 26/28] Template for config. --- .../modules/deadline/plugins/publish/submit_publish_job.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 02aa1043d18..89a4e5d377a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -552,7 +552,7 @@ def _create_instances_for_aov( "colorspace": colorspace, "config": { "path": additional_data["colorspaceConfig"], - "template": "" + "template": additional_data["colorspaceTemplate"] }, "display": additional_data["display"], "view": additional_data["view"] @@ -920,7 +920,10 @@ def process(self, instance): "renderProducts": instance.data["renderProducts"], "colorspaceConfig": instance.data["colorspaceConfig"], "display": instance.data["colorspaceDisplay"], - "view": instance.data["colorspaceView"] + "view": instance.data["colorspaceView"], + "colorspaceTemplate": instance.data["colorspaceConfig"].replace( + context.data["anatomy"].roots["work"], "{root[work]}" + ) } if isinstance(data.get("expectedFiles")[0], dict): From 7b6fb46cd191ab39ae43b9664365a0422203ad58 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 20 Feb 2023 16:05:24 +0000 Subject: [PATCH 27/28] Fix colorspaceTemplate --- openpype/modules/deadline/plugins/publish/submit_publish_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 89a4e5d377a..bd5933926c5 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -922,7 +922,7 @@ def process(self, instance): "display": instance.data["colorspaceDisplay"], "view": instance.data["colorspaceView"], "colorspaceTemplate": instance.data["colorspaceConfig"].replace( - context.data["anatomy"].roots["work"], "{root[work]}" + str(context.data["anatomy"].roots["work"]), "{root[work]}" ) } From 8f7b6c6a0e9f8b708e618fb5fec9277305786859 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 22 Feb 2023 06:58:20 +0000 Subject: [PATCH 28/28] Hound. --- openpype/modules/deadline/plugins/publish/submit_publish_job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 425c236b7ff..c651782392a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -962,7 +962,6 @@ def process(self, instance): "s" if len(instances) > 1 else "")) else: - #Need to inject colorspace here. representations = self._get_representations( instance_skeleton_data, data.get("expectedFiles")