From 6326719623095f4d6499d98f5856b894dd93f7f0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 21 Apr 2020 19:28:32 +0200 Subject: [PATCH 01/71] initial commit for review and burnin filtering --- pype/plugins/global/publish/extract_review.py | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 625c96566d4..c092ee4eeea 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1,10 +1,14 @@ import os +import re import pyblish.api import clique import pype.api import pype.lib +StringType = type("") + + class ExtractReview(pyblish.api.InstancePlugin): """Extracting Review mov file for Ftrack @@ -22,13 +26,238 @@ class ExtractReview(pyblish.api.InstancePlugin): families = ["review"] hosts = ["nuke", "maya", "shell"] + # Legacy attributes outputs = {} ext_filter = [] to_width = 1920 to_height = 1080 + # New attributes + profiles = None + def process(self, instance): + if self.profiles is None: + return self.legacy_process(instance) + + profile_filter_data = { + "host": pyblish.api.registered_hosts()[-1].title(), + "family": self.main_family_from_instance(instance), + "task": os.environ["AVALON_TASK"] + } + + profile = self.filter_profiles_by_data( + self.profiles, profile_filter_data + ) + if not profile: + return + + instance_families = self.families_from_instance(instance) + outputs = self.filter_outputs_by_families(profile, instance_families) + if not outputs: + return + + # TODO repre loop + repre_tags_low = [tag.lower() for tag in repre.get("tags", [])] + # Check tag filters + tag_filters = output_filters.get("tags") + if tag_filters: + tag_filters_low = [tag.lower() for tag in tag_filters] + valid = False + for tag in repre_tags_low: + if tag in tag_filters_low: + valid = True + break + + if not valid: + continue + + def main_family_from_instance(self, instance): + family = instance.data.get("family") + if not family: + family = instance.data["families"][0] + return family + + def families_from_instance(self, instance): + families = [] + family = instance.data.get("family") + if family: + families.append(family) + + for family in (instance.data.get("families") or tuple()): + if family not in families: + families.append(family) + return families + + def compile_list_of_regexes(self, in_list): + regexes = [] + if not in_list: + return regexes + + for item in in_list: + if not item: + continue + + if not isinstance(item, StringType): + self.log.warning(( + "Invalid type \"{}\" value \"{}\"." + " Expected . Skipping." + ).format(str(type(item)), str(item))) + continue + + regexes.append(re.compile(item)) + return regexes + + def validate_value_by_regexes(self, in_list, value): + """Validates in any regexe from list match entered value. + + Args: + in_list (list): List with regexes. + value (str): String where regexes is checked. + + Returns: + int: Returns `0` when list is not set or is empty. Returns `1` when + any regex match value and returns `-1` when none of regexes + match value entered. + """ + if not in_list: + return 0 + + output = -1 + regexes = self.compile_list_of_regexes(in_list) + for regex in regexes: + if re.match(regex, value): + output = 1 + break + return output + + def filter_profiles_by_data(self, profiles, filter_data): + """ Filter profiles by Host name, Task name and main Family. + + Filtering keys are "hosts" (list), "tasks" (list), "families" (list). + If key is not find or is empty than it's expected to match. + + Args: + profiles (list): Profiles definition from presets. + filter_data (dict): Dictionary with data for filtering. + Required keys are "host" - Host name, "task" - Task name + and "family" - Main . + """ + host_name = filter_data["host"] + task_name = filter_data["task"] + family = filter_data["family"] + + matching_profiles = None + highest_profile_points = -1 + # Each profile get 1 point for each matching filter. Profile with most + # points or first in row is returnd. + for profile in profiles: + profile_points = 0 + + # Host filtering + host_names = profile.get("hosts") + match = self.validate_value_by_regexes(host_names, host_name) + if match == -1: + continue + profile_points += match + + # Task filtering + task_names = profile.get("tasks") + match = self.validate_value_by_regexes(task_names, task_name) + if match == -1: + continue + profile_points += match + + # Family filtering + families = profile.get("families") + match = self.validate_value_by_regexes(families, family) + if match == -1: + continue + profile_points += match + + if profile_points == highest_profile_points: + matching_profiles.append(profile) + + elif profile_points > highest_profile_points: + highest_profile_points = profile_points + matching_profiles = [] + matching_profiles.append(profile) + + if not matching_profiles: + self.log.info(( + "None of profiles match your setup." + " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" + ).format(**filter_data)) + return + + if len(matching_profiles) > 1: + self.log.warning(( + "More than one profile match your setup." + " Using first found profile." + " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" + ).format(**filter_data)) + + return matching_profiles[0] + + def families_filter_validation(self, families, output_families_filter): + if not output_families_filter: + return True + + single_families = [] + combination_families = [] + for family_filter in output_families_filter: + if not family_filter: + continue + if isinstance(family_filter, (list, tuple)): + _family_filter = [] + for family in family_filter: + if family: + _family_filter.append(family.lower()) + combination_families.append(_family_filter) + else: + single_families.append(family_filter.lower()) + + for family in single_families: + if family in families: + return True + + for family_combination in combination_families: + valid = True + for family in family_combination: + if family not in families: + valid = False + break + + if valid: + return True + + return False + + def filter_outputs_by_families(self, profile, families): + outputs = profile.get("outputs") or [] + if not outputs: + return outputs + + # lower values + # QUESTION is this valid operation? + families = [family.lower() for family in families] + + filtered_outputs = {} + for filename_suffix, output_def in outputs.items(): + output_filters = output_def.get("output_filter") + # When filters not set then skip filtering process + if not output_filters: + filtered_outputs[filename_suffix] = output_def + continue + + families_filters = output_filters.get("families") + if not self.families_filter_validation(families, families_filters): + continue + + filtered_outputs[filename_suffix] = output_def + + return filtered_outputs + def legacy_process(self, instance): output_profiles = self.outputs or {} inst_data = instance.data From 60403273daef3e756c71c9c917428c8f6ab661bb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 11:18:16 +0200 Subject: [PATCH 02/71] filter_profiles_by_data renamed to find_matching_profile and swaped arguments in validate_value_by_regexes --- pype/plugins/global/publish/extract_review.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index c092ee4eeea..366745d8a8d 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -5,7 +5,6 @@ import pype.api import pype.lib - StringType = type("") @@ -26,16 +25,17 @@ class ExtractReview(pyblish.api.InstancePlugin): families = ["review"] hosts = ["nuke", "maya", "shell"] + # Preset attributes + profiles = None + # Legacy attributes outputs = {} ext_filter = [] to_width = 1920 to_height = 1080 - # New attributes - profiles = None - def process(self, instance): + # Use legacy processing when `profiles` is not set. if self.profiles is None: return self.legacy_process(instance) @@ -45,7 +45,7 @@ def process(self, instance): "task": os.environ["AVALON_TASK"] } - profile = self.filter_profiles_by_data( + profile = self.find_matching_profile( self.profiles, profile_filter_data ) if not profile: @@ -107,7 +107,7 @@ def compile_list_of_regexes(self, in_list): regexes.append(re.compile(item)) return regexes - def validate_value_by_regexes(self, in_list, value): + def validate_value_by_regexes(self, value, in_list): """Validates in any regexe from list match entered value. Args: @@ -130,7 +130,7 @@ def validate_value_by_regexes(self, in_list, value): break return output - def filter_profiles_by_data(self, profiles, filter_data): + def find_matching_profile(self, profiles, filter_data): """ Filter profiles by Host name, Task name and main Family. Filtering keys are "hosts" (list), "tasks" (list), "families" (list). @@ -155,21 +155,21 @@ def filter_profiles_by_data(self, profiles, filter_data): # Host filtering host_names = profile.get("hosts") - match = self.validate_value_by_regexes(host_names, host_name) + match = self.validate_value_by_regexes(host_name, host_names) if match == -1: continue profile_points += match # Task filtering task_names = profile.get("tasks") - match = self.validate_value_by_regexes(task_names, task_name) + match = self.validate_value_by_regexes(task_name, task_names) if match == -1: continue profile_points += match # Family filtering families = profile.get("families") - match = self.validate_value_by_regexes(families, family) + match = self.validate_value_by_regexes(family, families) if match == -1: continue profile_points += match From 06f0312191c0017722086a0557107f418aa7ba22 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 11:19:35 +0200 Subject: [PATCH 03/71] find_matching_profile has more complex system of getting most matching profile when more than one profile was filtered --- pype/plugins/global/publish/extract_review.py | 68 ++++++++++++++++--- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 366745d8a8d..3514339ae88 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -148,10 +148,13 @@ def find_matching_profile(self, profiles, filter_data): matching_profiles = None highest_profile_points = -1 + profile_values = {} # Each profile get 1 point for each matching filter. Profile with most - # points or first in row is returnd. + # points is returnd. For cases when more than one profile will match + # are also stored ordered lists of matching values. for profile in profiles: profile_points = 0 + profile_value = [] # Host filtering host_names = profile.get("hosts") @@ -159,6 +162,7 @@ def find_matching_profile(self, profiles, filter_data): if match == -1: continue profile_points += match + profile_value.append(bool(match)) # Task filtering task_names = profile.get("tasks") @@ -166,6 +170,7 @@ def find_matching_profile(self, profiles, filter_data): if match == -1: continue profile_points += match + profile_value.append(bool(match)) # Family filtering families = profile.get("families") @@ -173,7 +178,12 @@ def find_matching_profile(self, profiles, filter_data): if match == -1: continue profile_points += match + profile_value.append(bool(match)) + if profile_points < highest_profile_points: + continue + + profile["__value__"] = profile_value if profile_points == highest_profile_points: matching_profiles.append(profile) @@ -189,14 +199,56 @@ def find_matching_profile(self, profiles, filter_data): ).format(**filter_data)) return - if len(matching_profiles) > 1: - self.log.warning(( - "More than one profile match your setup." - " Using first found profile." - " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" - ).format(**filter_data)) + if len(matching_profiles) == 1: + # Pop temporary key `__value__` + matching_profiles[0].pop("__value__") + return matching_profiles[0] + + self.log.warning(( + "More than one profile match your setup." + " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" + ).format(**filter_data)) + + # Filter all profiles with highest points value. First filter profiles + # with matching host if there are any then filter profiles by task + # name if there are any and lastly filter by family. Else use first in + # list. + idx = 0 + final_profile = None + while True: + profiles_true = [] + profiles_false = [] + for profile in matching_profiles: + value = profile["__value__"] + # Just use first profile when idx is greater than values. + if not idx < len(value): + final_profile = profile + break + + if value[idx]: + profiles_true.append(profile) + else: + profiles_false.append(profile) - return matching_profiles[0] + if final_profile is not None: + break + + if profiles_true: + matching_profiles = profiles_true + else: + matching_profiles = profiles_false + + if len(matching_profiles) == 1: + final_profile = matching_profiles[0] + break + idx += 1 + + final_profile.pop("__value__") + self.log.info( + "Using first most matching profile in match order:" + " Host name -> Task name -> Family." + ) + return final_profile def families_filter_validation(self, families, output_families_filter): if not output_families_filter: From f84d9dc61646438d24d160c997897c0d7c51e13a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 11:19:59 +0200 Subject: [PATCH 04/71] added filter_outputs_by_tags to filter outputs per representation --- pype/plugins/global/publish/extract_review.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 3514339ae88..2e089fb7cf6 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -309,6 +309,31 @@ def filter_outputs_by_families(self, profile, families): return filtered_outputs + def filter_outputs_by_tags(self, outputs, tags): + filtered_outputs = {} + repre_tags_low = [tag.lower() for tag in tags] + for filename_suffix, output_def in outputs.values(): + valid = True + output_filters = output_def.get("output_filter") + if output_filters: + # Check tag filters + tag_filters = output_filters.get("tags") + if tag_filters: + tag_filters_low = [tag.lower() for tag in tag_filters] + valid = False + for tag in repre_tags_low: + if tag in tag_filters_low: + valid = True + break + + if not valid: + continue + + if valid: + filtered_outputs[filename_suffix] = output_def + + return filtered_outputs + def legacy_process(self, instance): output_profiles = self.outputs or {} From a4a1d0bea68fbadeffc70e600328a82c2191345e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 11:20:51 +0200 Subject: [PATCH 05/71] added first step of representation loop --- pype/plugins/global/publish/extract_review.py | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 2e089fb7cf6..1da9eb186ec 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -52,25 +52,44 @@ def process(self, instance): return instance_families = self.families_from_instance(instance) - outputs = self.filter_outputs_by_families(profile, instance_families) - if not outputs: + profile_outputs = self.filter_outputs_by_families(profile, instance_families) + if not profile_outputs: return - # TODO repre loop - repre_tags_low = [tag.lower() for tag in repre.get("tags", [])] - # Check tag filters - tag_filters = output_filters.get("tags") - if tag_filters: - tag_filters_low = [tag.lower() for tag in tag_filters] - valid = False - for tag in repre_tags_low: - if tag in tag_filters_low: - valid = True - break + context = instance.context + + fps = float(instance.data["fps"]) + frame_start = instance.data.get("frameStart") + frame_end = instance.data.get("frameEnd") + handle_start = instance.data.get( + "handleStart", + context.data.get("handleStart") + ) + handle_end = instance.data.get( + "handleEnd", + context.data.get("handleEnd") + ) + pixel_aspect = instance.data.get("pixelAspect", 1) + resolution_width = instance.data.get("resolutionWidth") + resolution_height = instance.data.get("resolutionHeight") + + ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - if not valid: + # get representation and loop them + representations = instance.data["representations"] + + for repre in tuple(representations): + tags = repre.get("tags", []) + if ( + "review" not in tags + or "multipartExr" in tags + or "thumbnail" in tags + ): continue + outputs = self.filter_outputs_by_tags(profile_outputs, tags) + if not outputs: + continue def main_family_from_instance(self, instance): family = instance.data.get("family") if not family: @@ -140,7 +159,11 @@ def find_matching_profile(self, profiles, filter_data): profiles (list): Profiles definition from presets. filter_data (dict): Dictionary with data for filtering. Required keys are "host" - Host name, "task" - Task name - and "family" - Main . + and "family" - Main instance family. + + Returns: + dict/None: Return most matching profile or None if none of profiles + match at least one criteria. """ host_name = filter_data["host"] task_name = filter_data["task"] From bf59ec9df72ebc02708cb8059648a911ddba098f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 17:23:41 +0200 Subject: [PATCH 06/71] added image/view exts and supported exts --- pype/plugins/global/publish/extract_review.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 1da9eb186ec..1b539b05a8b 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1,5 +1,6 @@ import os import re +import copy import pyblish.api import clique import pype.api @@ -24,6 +25,9 @@ class ExtractReview(pyblish.api.InstancePlugin): order = pyblish.api.ExtractorOrder + 0.02 families = ["review"] hosts = ["nuke", "maya", "shell"] + image_exts = ["exr", "jpg", "jpeg", "png", "dpx"] + video_exts = ["mov", "mp4"] + supported_exts = image_exts + video_exts # Preset attributes profiles = None From dfc32490202cc3bd0a3a4791a7d1a66da01c60f4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 17:24:13 +0200 Subject: [PATCH 07/71] first step of representations loop --- pype/plugins/global/publish/extract_review.py | 200 ++++++++++++++++-- 1 file changed, 178 insertions(+), 22 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 1b539b05a8b..620de20fcb0 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -27,6 +27,15 @@ class ExtractReview(pyblish.api.InstancePlugin): hosts = ["nuke", "maya", "shell"] image_exts = ["exr", "jpg", "jpeg", "png", "dpx"] video_exts = ["mov", "mp4"] + # image_exts = [".exr", ".jpg", ".jpeg", ".jpe", ".jif", ".jfif", ".jfi", ".png", ".dpx"] + # video_exts = [ + # ".webm", ".mkv", ".flv", ".flv", ".vob", ".ogv", ".ogg", ".drc", + # ".gif", ".gifv", ".mng", ".avi", ".MTS", ".M2TS", + # ".TS", ".mov", ".qt", ".wmv", ".yuv", ".rm", ".rmvb", + # ".asf", ".amv", ".mp4", ".m4p", ".mpg", ".mp2", ".mpeg", + # ".mpe", ".mpv", ".mpg", ".mpeg", ".m2v", ".m4v", ".svi", ".3gp", + # ".3g2", ".mxf", ".roq", ".nsv", ".flv", ".f4v", ".f4p", ".f4a", ".f4b" + # ] supported_exts = image_exts + video_exts # Preset attributes @@ -56,33 +65,18 @@ def process(self, instance): return instance_families = self.families_from_instance(instance) - profile_outputs = self.filter_outputs_by_families(profile, instance_families) + profile_outputs = self.filter_outputs_by_families( + profile, instance_families + ) if not profile_outputs: return - context = instance.context - - fps = float(instance.data["fps"]) - frame_start = instance.data.get("frameStart") - frame_end = instance.data.get("frameEnd") - handle_start = instance.data.get( - "handleStart", - context.data.get("handleStart") - ) - handle_end = instance.data.get( - "handleEnd", - context.data.get("handleEnd") - ) - pixel_aspect = instance.data.get("pixelAspect", 1) - resolution_width = instance.data.get("resolutionWidth") - resolution_height = instance.data.get("resolutionHeight") + instance_data = None ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - # get representation and loop them - representations = instance.data["representations"] - - for repre in tuple(representations): + # Loop through representations + for repre in tuple(instance.data["representations"]): tags = repre.get("tags", []) if ( "review" not in tags @@ -91,9 +85,171 @@ def process(self, instance): ): continue + source_ext = repre["ext"] + if source_ext.startswith("."): + source_ext = source_ext[1:] + + if source_ext not in self.supported_exts: + continue + + # Filter output definition by representation tags (optional) outputs = self.filter_outputs_by_tags(profile_outputs, tags) if not outputs: continue + + staging_dir = repre["stagingDir"] + + # Prepare instance data. + # NOTE Till this point it is not required to have set most + # of keys in instance data. So publishing won't crash if plugin + # won't get here and instance miss required keys. + if instance_data is None: + instance_data = self.prepare_instance_data(instance) + + for filename_suffix, output_def in outputs.items(): + + # Create copy of representation + new_repre = copy.deepcopy(repre) + + ext = output_def.get("ext") or "mov" + if ext.startswith("."): + ext = ext[1:] + + additional_tags = output_def.get("tags") or [] + # TODO new method? + # `self.new_repre_tags(new_repre, additional_tags)` + # Remove "delete" tag from new repre if there is + if "delete" in new_repre["tags"]: + new_repre["tags"].remove("delete") + + # Add additional tags from output definition to representation + for tag in additional_tags: + if tag not in new_repre["tags"]: + new_repre["tags"].append(tag) + + self.log.debug( + "New representation ext: \"{}\" | tags: `{}`".format( + ext, new_repre["tags"] + ) + ) + + # Output is image file sequence witht frames + # TODO change variable to `output_is_sequence` + # QUESTION Should we check for "sequence" only in additional + # tags or in all tags of new representation + is_sequence = ( + "sequence" in additional_tags + and (ext in self.image_exts) + ) + + # no handles switch from profile tags + no_handles = "no-handles" in additional_tags + + # TODO Find better way how to find out if input is sequence + # Theoretically issues: + # - there may be multiple files ant not be sequence + # - remainders are not checked at all + # - there can be more than one collection + if isinstance(repre["files"], (tuple, list)): + collections, remainder = clique.assemble(repre["files"]) + + full_input_path = os.path.join( + staging_dir, + collections[0].format("{head}{padding}{tail}") + ) + + filename = collections[0].format("{head}") + if filename.endswith("."): + filename = filename[:-1] + else: + full_input_path = os.path.join( + staging_dir, repre["files"] + ) + filename = os.path.splitext(repre["files"])[0] + + # QUESTION This breaks Anatomy template system is it ok? + # How do we care about multiple outputs with same extension? + if is_sequence: + filename_base = filename + "_{0}".format(filename_suffix) + repr_file = filename_base + ".%08d.{0}".format( + ext + ) + new_repre["sequence_file"] = repr_file + full_output_path = os.path.join( + staging_dir, filename_base, repr_file + ) + + else: + repr_file = filename + "_{0}.{1}".format( + filename_suffix, ext + ) + full_output_path = os.path.join(staging_dir, repr_file) + + self.log.info("Input path {}".format(full_input_path)) + self.log.info("Output path {}".format(full_output_path)) + + # QUESTION Why the hell we do this? + # add families + for tag in additional_tags: + if tag not in instance.data["families"]: + instance.data["families"].append(tag) + + # Get FFmpeg arguments from profile presets + output_ffmpeg_args = output_def.get("ffmpeg_args") or {} + output_ffmpeg_input = output_ffmpeg_args.get("input") or [] + output_ffmpeg_filters = output_ffmpeg_args.get("filters") or [] + output_ffmpeg_output = output_ffmpeg_args.get("output") or [] + + ffmpeg_input_args = [] + ffmpeg_output_args = [] + + # Override output file + ffmpeg_input_args.append("-y") + # Add input args from presets + ffmpeg_input_args.extend(output_ffmpeg_input) + + + if isinstance(repre["files"], list): + # QUESTION What is sence of this? + if frame_start_handle != repre.get( + "detectedStart", frame_start_handle + ): + frame_start_handle = repre.get("detectedStart") + + # exclude handle if no handles defined + if no_handles: + frame_start_handle = frame_start + frame_end_handle = frame_end + + ffmpeg_input_args.append( + "-start_number {0} -framerate {1}".format( + frame_start_handle, fps)) + else: + if no_handles: + start_sec = float(handle_start) / fps + ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) + frame_start_handle = frame_start + frame_end_handle = frame_end + + + def prepare_instance_data(self, instance): + return { + "fps": float(instance.data["fps"]), + "frame_start": instance.data["frameStart"], + "frame_end": instance.data["frameEnd"], + "handle_start": instance.data.get( + "handleStart", + instance.context.data["handleStart"] + ), + "handle_end": instance.data.get( + "handleEnd", + instance.context.data["handleEnd"] + ), + "pixel_aspect": instance.data.get("pixelAspect", 1), + "resolution_width": instance.data.get("resolutionWidth"), + "resolution_height": instance.data.get("resolutionHeight") + } + def main_family_from_instance(self, instance): family = instance.data.get("family") if not family: @@ -339,7 +495,7 @@ def filter_outputs_by_families(self, profile, families): def filter_outputs_by_tags(self, outputs, tags): filtered_outputs = {} repre_tags_low = [tag.lower() for tag in tags] - for filename_suffix, output_def in outputs.values(): + for filename_suffix, output_def in outputs.items(): valid = True output_filters = output_def.get("output_filter") if output_filters: From 0586fff9ab1bda66884cc7114ee36508ed3a955c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 22 Apr 2020 19:07:27 +0200 Subject: [PATCH 08/71] few more steps and added few comments --- pype/plugins/global/publish/extract_review.py | 114 +++++++++++------- 1 file changed, 69 insertions(+), 45 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 620de20fcb0..87cb519485d 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -111,13 +111,13 @@ def process(self, instance): # Create copy of representation new_repre = copy.deepcopy(repre) - ext = output_def.get("ext") or "mov" - if ext.startswith("."): - ext = ext[1:] + output_ext = output_def.get("ext") or "mov" + if output_ext.startswith("."): + output_ext = output_ext[1:] additional_tags = output_def.get("tags") or [] # TODO new method? - # `self.new_repre_tags(new_repre, additional_tags)` + # `self.prepare_new_repre_tags(new_repre, additional_tags)` # Remove "delete" tag from new repre if there is if "delete" in new_repre["tags"]: new_repre["tags"].remove("delete") @@ -129,24 +129,28 @@ def process(self, instance): self.log.debug( "New representation ext: \"{}\" | tags: `{}`".format( - ext, new_repre["tags"] + output_ext, new_repre["tags"] ) ) # Output is image file sequence witht frames # TODO change variable to `output_is_sequence` + # QUESTION Shall we do it in opposite? Expect that if output + # extension is image format and input is sequence or video + # format then do sequence and single frame only if tag is + # "single-frame" (or similar) # QUESTION Should we check for "sequence" only in additional # tags or in all tags of new representation is_sequence = ( "sequence" in additional_tags - and (ext in self.image_exts) + and (output_ext in self.image_exts) ) # no handles switch from profile tags no_handles = "no-handles" in additional_tags - # TODO Find better way how to find out if input is sequence - # Theoretically issues: + # TODO GLOBAL ISSUE - Find better way how to find out if input + # is sequence. Issues( in theory): # - there may be multiple files ant not be sequence # - remainders are not checked at all # - there can be more than one collection @@ -168,11 +172,14 @@ def process(self, instance): filename = os.path.splitext(repre["files"])[0] # QUESTION This breaks Anatomy template system is it ok? - # How do we care about multiple outputs with same extension? + # QUESTION How do we care about multiple outputs with same + # extension? (Expect we don't...) + # - possible solution add "<{review_suffix}>" into templates + # but that may cause issues when clients remove that. if is_sequence: filename_base = filename + "_{0}".format(filename_suffix) repr_file = filename_base + ".%08d.{0}".format( - ext + output_ext ) new_repre["sequence_file"] = repr_file full_output_path = os.path.join( @@ -181,7 +188,7 @@ def process(self, instance): else: repr_file = filename + "_{0}.{1}".format( - filename_suffix, ext + filename_suffix, output_ext ) full_output_path = os.path.join(staging_dir, repr_file) @@ -194,42 +201,59 @@ def process(self, instance): if tag not in instance.data["families"]: instance.data["families"].append(tag) - # Get FFmpeg arguments from profile presets - output_ffmpeg_args = output_def.get("ffmpeg_args") or {} - output_ffmpeg_input = output_ffmpeg_args.get("input") or [] - output_ffmpeg_filters = output_ffmpeg_args.get("filters") or [] - output_ffmpeg_output = output_ffmpeg_args.get("output") or [] - - ffmpeg_input_args = [] - ffmpeg_output_args = [] - - # Override output file - ffmpeg_input_args.append("-y") - # Add input args from presets - ffmpeg_input_args.extend(output_ffmpeg_input) - + ffmpeg_args = self._ffmpeg_arguments( + output_def, instance, instance_data + ) - if isinstance(repre["files"], list): - # QUESTION What is sence of this? - if frame_start_handle != repre.get( - "detectedStart", frame_start_handle - ): - frame_start_handle = repre.get("detectedStart") + def _ffmpeg_arguments(output_def, instance, repre, instance_data): + # TODO split into smaller methods and use these variable only there + fps = instance_data["fps"] + frame_start = instance_data["frame_start"] + frame_end = instance_data["frame_end"] + handle_start = instance_data["handle_start"] + handle_end = instance_data["handle_end"] + frame_start_handle = frame_start - handle_start, + frame_end_handle = frame_end + handle_end, + pixel_aspect = instance_data["pixel_aspect"] + resolution_width = instance_data["resolution_width"] + resolution_height = instance_data["resolution_height"] + + # Get FFmpeg arguments from profile presets + output_ffmpeg_args = output_def.get("ffmpeg_args") or {} + output_ffmpeg_input = output_ffmpeg_args.get("input") or [] + output_ffmpeg_filters = output_ffmpeg_args.get("filters") or [] + output_ffmpeg_output = output_ffmpeg_args.get("output") or [] + + ffmpeg_input_args = [] + ffmpeg_output_args = [] + + # Override output file + ffmpeg_input_args.append("-y") + # Add input args from presets + ffmpeg_input_args.extend(output_ffmpeg_input) + + if isinstance(repre["files"], list): + # QUESTION What is sence of this? + if frame_start_handle != repre.get( + "detectedStart", frame_start_handle + ): + frame_start_handle = repre.get("detectedStart") - # exclude handle if no handles defined - if no_handles: - frame_start_handle = frame_start - frame_end_handle = frame_end + # exclude handle if no handles defined + if no_handles: + frame_start_handle = frame_start + frame_end_handle = frame_end - ffmpeg_input_args.append( - "-start_number {0} -framerate {1}".format( - frame_start_handle, fps)) - else: - if no_handles: - start_sec = float(handle_start) / fps - ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) - frame_start_handle = frame_start - frame_end_handle = frame_end + ffmpeg_input_args.append( + "-start_number {0} -framerate {1}".format( + frame_start_handle, fps)) + else: + if no_handles: + # QUESTION why we are using seconds instead of frames? + start_sec = float(handle_start) / fps + ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) + frame_start_handle = frame_start + frame_end_handle = frame_end def prepare_instance_data(self, instance): @@ -247,7 +271,7 @@ def prepare_instance_data(self, instance): ), "pixel_aspect": instance.data.get("pixelAspect", 1), "resolution_width": instance.data.get("resolutionWidth"), - "resolution_height": instance.data.get("resolutionHeight") + "resolution_height": instance.data.get("resolutionHeight"), } def main_family_from_instance(self, instance): From 9ed201c0cd1e8d278d38fa2602374966f43985c2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 23 Apr 2020 11:59:13 +0200 Subject: [PATCH 09/71] process is splitted more than already was --- pype/plugins/global/publish/extract_review.py | 479 +++++++++++++----- 1 file changed, 340 insertions(+), 139 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 87cb519485d..887c5067018 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -65,13 +65,17 @@ def process(self, instance): return instance_families = self.families_from_instance(instance) - profile_outputs = self.filter_outputs_by_families( + _profile_outputs = self.filter_outputs_by_families( profile, instance_families ) - if not profile_outputs: + if not _profile_outputs: return - instance_data = None + # Store `filename_suffix` to save to save arguments + profile_outputs = [] + for filename_suffix, definition in _profile_outputs.items(): + definition["filename_suffix"] = filename_suffix + profile_outputs.append(definition) ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") @@ -97,24 +101,14 @@ def process(self, instance): if not outputs: continue - staging_dir = repre["stagingDir"] - # Prepare instance data. # NOTE Till this point it is not required to have set most # of keys in instance data. So publishing won't crash if plugin # won't get here and instance miss required keys. - if instance_data is None: - instance_data = self.prepare_instance_data(instance) - - for filename_suffix, output_def in outputs.items(): - + for output_def in outputs: # Create copy of representation new_repre = copy.deepcopy(repre) - output_ext = output_def.get("ext") or "mov" - if output_ext.startswith("."): - output_ext = output_ext[1:] - additional_tags = output_def.get("tags") or [] # TODO new method? # `self.prepare_new_repre_tags(new_repre, additional_tags)` @@ -128,152 +122,359 @@ def process(self, instance): new_repre["tags"].append(tag) self.log.debug( - "New representation ext: \"{}\" | tags: `{}`".format( - output_ext, new_repre["tags"] - ) - ) - - # Output is image file sequence witht frames - # TODO change variable to `output_is_sequence` - # QUESTION Shall we do it in opposite? Expect that if output - # extension is image format and input is sequence or video - # format then do sequence and single frame only if tag is - # "single-frame" (or similar) - # QUESTION Should we check for "sequence" only in additional - # tags or in all tags of new representation - is_sequence = ( - "sequence" in additional_tags - and (output_ext in self.image_exts) + "New representation tags: `{}`".format(new_repre["tags"]) ) - # no handles switch from profile tags - no_handles = "no-handles" in additional_tags - - # TODO GLOBAL ISSUE - Find better way how to find out if input - # is sequence. Issues( in theory): - # - there may be multiple files ant not be sequence - # - remainders are not checked at all - # - there can be more than one collection - if isinstance(repre["files"], (tuple, list)): - collections, remainder = clique.assemble(repre["files"]) - - full_input_path = os.path.join( - staging_dir, - collections[0].format("{head}{padding}{tail}") - ) - - filename = collections[0].format("{head}") - if filename.endswith("."): - filename = filename[:-1] - else: - full_input_path = os.path.join( - staging_dir, repre["files"] - ) - filename = os.path.splitext(repre["files"])[0] - - # QUESTION This breaks Anatomy template system is it ok? - # QUESTION How do we care about multiple outputs with same - # extension? (Expect we don't...) - # - possible solution add "<{review_suffix}>" into templates - # but that may cause issues when clients remove that. - if is_sequence: - filename_base = filename + "_{0}".format(filename_suffix) - repr_file = filename_base + ".%08d.{0}".format( - output_ext - ) - new_repre["sequence_file"] = repr_file - full_output_path = os.path.join( - staging_dir, filename_base, repr_file - ) - - else: - repr_file = filename + "_{0}.{1}".format( - filename_suffix, output_ext - ) - full_output_path = os.path.join(staging_dir, repr_file) - - self.log.info("Input path {}".format(full_input_path)) - self.log.info("Output path {}".format(full_output_path)) - - # QUESTION Why the hell we do this? + # QUESTION Why the hell we do this, adding tags to families? # add families for tag in additional_tags: if tag not in instance.data["families"]: instance.data["families"].append(tag) - ffmpeg_args = self._ffmpeg_arguments( - output_def, instance, instance_data - ) - - def _ffmpeg_arguments(output_def, instance, repre, instance_data): - # TODO split into smaller methods and use these variable only there - fps = instance_data["fps"] - frame_start = instance_data["frame_start"] - frame_end = instance_data["frame_end"] - handle_start = instance_data["handle_start"] - handle_end = instance_data["handle_end"] - frame_start_handle = frame_start - handle_start, - frame_end_handle = frame_end + handle_end, - pixel_aspect = instance_data["pixel_aspect"] - resolution_width = instance_data["resolution_width"] - resolution_height = instance_data["resolution_height"] - + ffmpeg_args = self._ffmpeg_arguments(output_def, instance) + + def repre_has_sequence(self, repre): + # TODO GLOBAL ISSUE - Find better way how to find out if input + # is sequence. Issues( in theory): + # - there may be multiple files ant not be sequence + # - remainders are not checked at all + # - there can be more than one collection + return isinstance(repre["files"], (list, tuple)) + + def _ffmpeg_arguments(self, output_def, instance, repre): + temp_data = self.prepare_temp_data(instance) + + # NOTE used different key for final frame start/end to not confuse + # those who don't know what + # - e.g. "frame_start_output" + # QUESTION should we use tags ONLY from output definition? + # - In that case `output_def.get("tags") or []` should replace + # `repre["tags"]`. + # Change output frames when output should be without handles + no_handles = "no-handles" in repre["tags"] + if no_handles: + temp_data["output_frame_start"] = temp_data["frame_start"] + temp_data["output_frame_end"] = temp_data["frame_end"] + + # TODO this may hold class which may be easier to work with # Get FFmpeg arguments from profile presets - output_ffmpeg_args = output_def.get("ffmpeg_args") or {} - output_ffmpeg_input = output_ffmpeg_args.get("input") or [] - output_ffmpeg_filters = output_ffmpeg_args.get("filters") or [] - output_ffmpeg_output = output_ffmpeg_args.get("output") or [] + out_def_ffmpeg_args = output_def.get("ffmpeg_args") or {} - ffmpeg_input_args = [] - ffmpeg_output_args = [] + ffmpeg_input_args = out_def_ffmpeg_args.get("input") or [] + ffmpeg_output_args = out_def_ffmpeg_args.get("output") or [] + ffmpeg_video_filters = out_def_ffmpeg_args.get("video_filters") or [] + ffmpeg_audio_filters = out_def_ffmpeg_args.get("audio_filters") or [] - # Override output file + # Add argument to override output file ffmpeg_input_args.append("-y") - # Add input args from presets - ffmpeg_input_args.extend(output_ffmpeg_input) - if isinstance(repre["files"], list): - # QUESTION What is sence of this? - if frame_start_handle != repre.get( - "detectedStart", frame_start_handle - ): - frame_start_handle = repre.get("detectedStart") + if no_handles: + # NOTE used `-frames:v` instead of `-t` + duration_frames = ( + temp_data["output_frame_end"] + - temp_data["output_frame_start"] + + 1 + ) + ffmpeg_output_args.append("-frames:v {}".format(duration_frames)) - # exclude handle if no handles defined - if no_handles: - frame_start_handle = frame_start - frame_end_handle = frame_end + if self.repre_has_sequence(repre): + # NOTE removed "detectedStart" key handling (NOT SET) + # Set start frame ffmpeg_input_args.append( - "-start_number {0} -framerate {1}".format( - frame_start_handle, fps)) - else: - if no_handles: - # QUESTION why we are using seconds instead of frames? - start_sec = float(handle_start) / fps - ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) - frame_start_handle = frame_start - frame_end_handle = frame_end + "-start_number {}".format(temp_data["output_frame_start"]) + ) + + # TODO add fps mapping `{fps: fraction}` + # - e.g.: { + # "25": "25/1", + # "24": "24/1", + # "23.976": "24000/1001" + # } + # Add framerate to input when input is sequence + ffmpeg_input_args.append( + "-framerate {}".format(temp_data["fps"]) + ) + + elif no_handles: + # QUESTION Shall we change this to use filter: + # `select="gte(n\,handle_start),setpts=PTS-STARTPTS` + # Pros: + # 1.) Python is not good at float operation + # 2.) FPS on instance may not be same as input's + start_sec = float(temp_data["handle_start"]) / temp_data["fps"] + ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) + + full_input_path, full_output_path = self.input_output_paths( + repre, output_def + ) + ffmpeg_input_args.append("-i \"{}\"".format(full_input_path)) + # Add audio arguments if there are any + audio_in_args, audio_filters, audio_out_args = self.audio_args( + instance, temp_data + ) + ffmpeg_input_args.extend(audio_in_args) + ffmpeg_audio_filters.extend(audio_filters) + ffmpeg_output_args.extend(audio_out_args) + + # In case audio is longer than video. + # QUESTION what if audio is shoter than video? + if "-shortest" not in ffmpeg_output_args: + ffmpeg_output_args.append("-shortest") + + ffmpeg_output_args.append("\"{}\"".format(full_output_path)) + + def prepare_temp_data(self, instance): + frame_start = instance.data["frameStart"] + handle_start = instance.data.get( + "handleStart", + instance.context.data["handleStart"] + ) + frame_end = instance.data["frameEnd"] + handle_end = instance.data.get( + "handleEnd", + instance.context.data["handleEnd"] + ) + + frame_start_handle = frame_start - handle_start + frame_end_handle = frame_end + handle_end - def prepare_instance_data(self, instance): return { "fps": float(instance.data["fps"]), - "frame_start": instance.data["frameStart"], - "frame_end": instance.data["frameEnd"], - "handle_start": instance.data.get( - "handleStart", - instance.context.data["handleStart"] - ), - "handle_end": instance.data.get( - "handleEnd", - instance.context.data["handleEnd"] - ), + "frame_start": frame_start, + "frame_end": frame_end, + "handle_start": handle_start, + "handle_end": handle_end, + "frame_start_handle": frame_start_handle, + "frame_end_handle": frame_end_handle, + "output_frame_start": frame_start_handle, + "output_frame_end": frame_end_handle, "pixel_aspect": instance.data.get("pixelAspect", 1), "resolution_width": instance.data.get("resolutionWidth"), "resolution_height": instance.data.get("resolutionHeight"), } + def input_output_paths(self, repre, output_def): + staging_dir = repre["stagingDir"] + + # TODO Define if extension should have dot or not + output_ext = output_def.get("ext") or "mov" + if output_ext.startswith("."): + output_ext = output_ext[1:] + + self.log.debug( + "New representation ext: `{}`".format(output_ext) + ) + + # Output is image file sequence witht frames + # QUESTION Shall we do it in opposite? Expect that if output + # extension is image format and input is sequence or video + # format then do sequence and single frame only if tag is + # "single-frame" (or similar) + # QUESTION should we use tags ONLY from output definition? + # - In that case `output_def.get("tags") or []` should replace + # `repre["tags"]`. + output_is_sequence = ( + "sequence" in repre["tags"] + and (output_ext in self.image_exts) + ) + + if self.repre_has_sequence(repre): + collections, remainder = clique.assemble(repre["files"]) + + full_input_path = os.path.join( + staging_dir, + collections[0].format("{head}{padding}{tail}") + ) + + filename = collections[0].format("{head}") + if filename.endswith("."): + filename = filename[:-1] + else: + full_input_path = os.path.join( + staging_dir, repre["files"] + ) + filename = os.path.splitext(repre["files"])[0] + + filename_suffix = output_def["filename_suffix"] + # QUESTION This breaks Anatomy template system is it ok? + # QUESTION How do we care about multiple outputs with same + # extension? (Expect we don't...) + # - possible solution add "<{review_suffix}>" into templates + # but that may cause issues when clients remove that (and it's + # ugly). + if output_is_sequence: + filename_base = "{}_{}".format( + filename, filename_suffix + ) + repr_file = "{}.%08d.{}".format( + filename_base, output_ext + ) + + repre["sequence_file"] = repr_file + full_output_path = os.path.join( + staging_dir, filename_base, repr_file + ) + + else: + repr_file = "{}_{}.{}".format( + filename, filename_suffix, output_ext + ) + full_output_path = os.path.join(staging_dir, repr_file) + + self.log.debug("Input path {}".format(full_input_path)) + self.log.debug("Output path {}".format(full_output_path)) + + return full_input_path, full_output_path + + def audio_args(self, instance, temp_data): + audio_in_args = [] + audio_filters = [] + audio_out_args = [] + audio_inputs = instance.data.get("audio") + if not audio_inputs: + return audio_in_args, audio_filters, audio_out_args + + for audio in audio_inputs: + # NOTE modified, always was expected "frameStartFtrack" which is + # STANGE?!!! + # TODO use different frame start! + offset_seconds = 0 + frame_start_ftrack = instance.data.get("frameStartFtrack") + if frame_start_ftrack is not None: + offset_frames = frame_start_ftrack - audio["offset"] + offset_seconds = offset_frames / temp_data["fps"] + + if offset_seconds > 0: + audio_in_args.append( + "-ss {}".format(offset_seconds) + ) + elif offset_seconds < 0: + audio_in_args.append( + "-itsoffset {}".format(abs(offset_seconds)) + ) + + audio_in_args.append("-i \"{}\"".format(audio["filename"])) + + # NOTE: These were changed from input to output arguments. + # NOTE: value in "-ac" was hardcoded to 2, changed to audio inputs len. + # Need to merge audio if there are more than 1 input. + if len(audio_inputs) > 1: + audio_out_args.append("-filter_complex amerge") + audio_out_args.append("-ac {}".format(len(audio_inputs))) + + return audio_in_args, audio_filters, audio_out_args + + def resolution_ratios(self, temp_data, output_def, repre): + output_width = output_def.get("width") + output_height = output_def.get("height") + output_pixel_aspect = output_def.get("aspect_ratio") + output_letterbox = output_def.get("letter_box") + + # defining image ratios + resolution_ratio = ( + (float(resolution_width) * pixel_aspect) / resolution_height + ) + delivery_ratio = float(self.to_width) / float(self.to_height) + self.log.debug("resolution_ratio: `{}`".format(resolution_ratio)) + self.log.debug("delivery_ratio: `{}`".format(delivery_ratio)) + + # shorten two decimals long float number for testing conditions + resolution_ratio_test = float("{:0.2f}".format(resolution_ratio)) + delivery_ratio_test = float("{:0.2f}".format(delivery_ratio)) + + # get scale factor + if resolution_ratio_test < delivery_ratio_test: + scale_factor = ( + float(self.to_width) / (resolution_width * pixel_aspect) + ) + else: + scale_factor = ( + float(self.to_height) / (resolution_height * pixel_aspect) + ) + + self.log.debug("__ scale_factor: `{}`".format(scale_factor)) + + filters = [] + # letter_box + if output_letterbox: + ffmpeg_width = self.to_width + ffmpeg_height = self.to_height + if "reformat" not in repre["tags"]: + output_letterbox /= pixel_aspect + if resolution_ratio_test != delivery_ratio_test: + ffmpeg_width = resolution_width + ffmpeg_height = int( + resolution_height * pixel_aspect) + else: + if resolution_ratio_test != delivery_ratio_test: + output_letterbox /= scale_factor + else: + output_letterbox /= pixel_aspect + + filters.append( + "scale={}x{}:flags=lanczos".format(ffmpeg_width, ffmpeg_height) + ) + # QUESTION shouldn't this contain aspect ration value instead of 1? + filters.append( + "setsar=1" + ) + filters.append(( + "drawbox=0:0:iw:round((ih-(iw*(1/{})))/2):t=fill:c=black" + ).format(output_letterbox)) + + filters.append(( + "drawbox=0:ih-round((ih-(iw*(1/{0})))/2)" + ":iw:round((ih-(iw*(1/{0})))/2):t=fill:c=black" + ).format(output_letterbox)) + + self.log.debug("pixel_aspect: `{}`".format(pixel_aspect)) + self.log.debug("resolution_width: `{}`".format(resolution_width)) + self.log.debug("resolution_height: `{}`".format(resolution_height)) + + # scaling none square pixels and 1920 width + # QUESTION: again check only output tags or repre tags + # WARNING: Duplication of filters when letter_box is set (or not?) + if "reformat" in repre["tags"]: + if resolution_ratio_test < delivery_ratio_test: + self.log.debug("lower then delivery") + width_scale = int(self.to_width * scale_factor) + width_half_pad = int((self.to_width - width_scale) / 2) + height_scale = self.to_height + height_half_pad = 0 + else: + self.log.debug("heigher then delivery") + width_scale = self.to_width + width_half_pad = 0 + scale_factor = ( + float(self.to_width) + / (float(resolution_width) * pixel_aspect) + ) + self.log.debug( + "__ scale_factor: `{}`".format(scale_factor) + ) + height_scale = int(resolution_height * scale_factor) + height_half_pad = int((self.to_height - height_scale) / 2) + + self.log.debug("width_scale: `{}`".format(width_scale)) + self.log.debug("width_half_pad: `{}`".format(width_half_pad)) + self.log.debug("height_scale: `{}`".format(height_scale)) + self.log.debug("height_half_pad: `{}`".format(height_half_pad)) + + filters.append( + "scale={}x{}:flags=lanczos".format(width_scale, height_scale) + ) + filters.append( + "pad={}:{}:{}:{}:black".format( + self.to_width, self.to_height, + width_half_pad, + height_half_pad + ) + filters.append("setsar=1") + + return filters + def main_family_from_instance(self, instance): family = instance.data.get("family") if not family: @@ -517,9 +718,9 @@ def filter_outputs_by_families(self, profile, families): return filtered_outputs def filter_outputs_by_tags(self, outputs, tags): - filtered_outputs = {} + filtered_outputs = [] repre_tags_low = [tag.lower() for tag in tags] - for filename_suffix, output_def in outputs.items(): + for output_def in outputs: valid = True output_filters = output_def.get("output_filter") if output_filters: @@ -537,7 +738,7 @@ def filter_outputs_by_tags(self, outputs, tags): continue if valid: - filtered_outputs[filename_suffix] = output_def + filtered_outputs.append(output_def) return filtered_outputs From cc153acb32f2771921cdc7d96c4320be65d23458 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 23 Apr 2020 15:39:25 +0200 Subject: [PATCH 10/71] all except reformatting should work --- pype/plugins/global/publish/extract_review.py | 310 +++++++++++++----- 1 file changed, 234 insertions(+), 76 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 887c5067018..170193ff122 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -25,19 +25,15 @@ class ExtractReview(pyblish.api.InstancePlugin): order = pyblish.api.ExtractorOrder + 0.02 families = ["review"] hosts = ["nuke", "maya", "shell"] + + # Supported extensions image_exts = ["exr", "jpg", "jpeg", "png", "dpx"] video_exts = ["mov", "mp4"] - # image_exts = [".exr", ".jpg", ".jpeg", ".jpe", ".jif", ".jfif", ".jfi", ".png", ".dpx"] - # video_exts = [ - # ".webm", ".mkv", ".flv", ".flv", ".vob", ".ogv", ".ogg", ".drc", - # ".gif", ".gifv", ".mng", ".avi", ".MTS", ".M2TS", - # ".TS", ".mov", ".qt", ".wmv", ".yuv", ".rm", ".rmvb", - # ".asf", ".amv", ".mp4", ".m4p", ".mpg", ".mp2", ".mpeg", - # ".mpe", ".mpv", ".mpg", ".mpeg", ".m2v", ".m4v", ".svi", ".3gp", - # ".3g2", ".mxf", ".roq", ".nsv", ".flv", ".f4v", ".f4p", ".f4a", ".f4b" - # ] supported_exts = image_exts + video_exts + # Path to ffmpeg + path_to_ffmpeg = None + # Preset attributes profiles = None @@ -52,6 +48,16 @@ def process(self, instance): if self.profiles is None: return self.legacy_process(instance) + # Run processing + self.main_process(instance) + + # Make sure cleanup happens and pop representations with "delete" tag. + for repre in tuple(instance.data["representations"]): + tags = repre.get("tags") or [] + if "delete" if tags: + instance.data["representations"].remove(repre) + + def main_process(self, instance): profile_filter_data = { "host": pyblish.api.registered_hosts()[-1].title(), "family": self.main_family_from_instance(instance), @@ -71,13 +77,14 @@ def process(self, instance): if not _profile_outputs: return - # Store `filename_suffix` to save to save arguments + # Store `filename_suffix` to save arguments profile_outputs = [] for filename_suffix, definition in _profile_outputs.items(): definition["filename_suffix"] = filename_suffix profile_outputs.append(definition) - ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + if self.path_to_ffmpeg is None: + self.path_to_ffmpeg = pype.lib.get_ffmpeg_tool_path("ffmpeg") # Loop through representations for repre in tuple(instance.data["representations"]): @@ -131,9 +138,39 @@ def process(self, instance): if tag not in instance.data["families"]: instance.data["families"].append(tag) - ffmpeg_args = self._ffmpeg_arguments(output_def, instance) + temp_data = self.prepare_temp_data(instance, repre, new_repre) + + ffmpeg_args = self._ffmpeg_arguments( + output_def, instance, temp_data + ) + subprcs_cmd = " ".join(ffmpeg_args) + + # run subprocess + self.log.debug("Executing: {}".format(subprcs_cmd)) + output = pype.api.subprocess(subprcs_cmd) + self.log.debug("Output: {}".format(output)) + + output_name = output_def["filename_suffix"] + if temp_data["without_handles"]: + output_name += "_noHandles" - def repre_has_sequence(self, repre): + new_repre.update({ + "name": output_def["filename_suffix"], + "outputName": output_name, + "outputDef": output_def, + "frameStartFtrack": temp_data["output_frame_start"], + "frameEndFtrack": temp_data["output_frame_end"] + }) + + # Force to pop these key if are in new repre + new_repre.pop("preview", None) + new_repre.pop("thumbnail", None) + + # adding representation + self.log.debug("Adding: {}".format(new_repre)) + instance.data["representations"].append(new_repre) + + def source_is_sequence(self, repre): # TODO GLOBAL ISSUE - Find better way how to find out if input # is sequence. Issues( in theory): # - there may be multiple files ant not be sequence @@ -141,21 +178,7 @@ def repre_has_sequence(self, repre): # - there can be more than one collection return isinstance(repre["files"], (list, tuple)) - def _ffmpeg_arguments(self, output_def, instance, repre): - temp_data = self.prepare_temp_data(instance) - - # NOTE used different key for final frame start/end to not confuse - # those who don't know what - # - e.g. "frame_start_output" - # QUESTION should we use tags ONLY from output definition? - # - In that case `output_def.get("tags") or []` should replace - # `repre["tags"]`. - # Change output frames when output should be without handles - no_handles = "no-handles" in repre["tags"] - if no_handles: - temp_data["output_frame_start"] = temp_data["frame_start"] - temp_data["output_frame_end"] = temp_data["frame_end"] - + def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): # TODO this may hold class which may be easier to work with # Get FFmpeg arguments from profile presets out_def_ffmpeg_args = output_def.get("ffmpeg_args") or {} @@ -168,8 +191,8 @@ def _ffmpeg_arguments(self, output_def, instance, repre): # Add argument to override output file ffmpeg_input_args.append("-y") - if no_handles: - # NOTE used `-frames:v` instead of `-t` + if temp_data["without_handles"]: + # NOTE used `-frames:v` instead of `-t` - should work the same way duration_frames = ( temp_data["output_frame_end"] - temp_data["output_frame_start"] @@ -177,7 +200,7 @@ def _ffmpeg_arguments(self, output_def, instance, repre): ) ffmpeg_output_args.append("-frames:v {}".format(duration_frames)) - if self.repre_has_sequence(repre): + if temp_data["source_is_sequence"]: # NOTE removed "detectedStart" key handling (NOT SET) # Set start frame @@ -196,7 +219,7 @@ def _ffmpeg_arguments(self, output_def, instance, repre): "-framerate {}".format(temp_data["fps"]) ) - elif no_handles: + elif temp_data["without_handles"]: # QUESTION Shall we change this to use filter: # `select="gte(n\,handle_start),setpts=PTS-STARTPTS` # Pros: @@ -206,7 +229,7 @@ def _ffmpeg_arguments(self, output_def, instance, repre): ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) full_input_path, full_output_path = self.input_output_paths( - repre, output_def + new_repre, output_def, temp_data ) ffmpeg_input_args.append("-i \"{}\"".format(full_input_path)) @@ -218,14 +241,80 @@ def _ffmpeg_arguments(self, output_def, instance, repre): ffmpeg_audio_filters.extend(audio_filters) ffmpeg_output_args.extend(audio_out_args) - # In case audio is longer than video. + # In case audio is longer than video`. # QUESTION what if audio is shoter than video? if "-shortest" not in ffmpeg_output_args: ffmpeg_output_args.append("-shortest") + res_filters = self.resolution_ratios(temp_data, output_def, new_repre) + ffmpeg_video_filters.extend(res_filters) + + ffmpeg_input_args = self.split_ffmpeg_args(ffmpeg_input_args) + + lut_filters = self.lut_filters(new_repre, instance, ffmpeg_input_args) + ffmpeg_video_filters.extend(lut_filters) + + # WARNING This must be latest added item to output arguments. ffmpeg_output_args.append("\"{}\"".format(full_output_path)) - def prepare_temp_data(self, instance): + return self.ffmpeg_full_args( + ffmpeg_input_args, + ffmpeg_video_filters, + ffmpeg_audio_filters, + ffmpeg_output_args + ) + + def split_ffmpeg_args(self, in_args): + splitted_args = [] + for arg in in_args: + sub_args = arg.split(" -") + if len(sub_args) == 1: + if arg and arg not in splitted_args: + splitted_args.append(arg) + continue + + for idx, arg in enumerate(sub_args): + if idx != 0: + arg = "-" + arg + + if arg and arg not in splitted_args: + splitted_args.append(arg) + return splitted_args + + def ffmpeg_full_args( + self, input_args, video_filters, audio_filters, output_args + ): + output_args = self.split_ffmpeg_args(output_args) + + video_args_dentifiers = ["-vf", "-filter:v"] + audio_args_dentifiers = ["-af", "-filter:a"] + for arg in tuple(output_args): + for identifier in video_args_dentifiers: + if identifier in arg: + output_args.remove(arg) + arg = arg.replace(identifier, "").strip() + video_filters.append(arg) + + for identifier in audio_args_dentifiers: + if identifier in arg: + output_args.remove(arg) + arg = arg.replace(identifier, "").strip() + audio_filters.append(arg) + + all_args = [] + all_args.append(self.path_to_ffmpeg) + all_args.extend(input_args) + if video_filters: + all_args.append("-filter:v {}".format(",".join(video_filters))) + + if audio_filters: + all_args.append("-filter:a {}".format(",".join(audio_filters))) + + all_args.extend(output_args) + + return all_args + + def prepare_temp_data(self, instance, repre, new_repre): frame_start = instance.data["frameStart"] handle_start = instance.data.get( "handleStart", @@ -240,6 +329,21 @@ def prepare_temp_data(self, instance): frame_start_handle = frame_start - handle_start frame_end_handle = frame_end + handle_end + # NOTE used different key for final frame start/end to not confuse + # those who don't know what + # - e.g. "frame_start_output" + # QUESTION should we use tags ONLY from output definition? + # - In that case `output_def.get("tags") or []` should replace + # `repre["tags"]`. + # Change output frames when output should be without handles + without_handles = "no-handles" in new_repre["tags"] + if without_handles: + output_frame_start = frame_start + output_frame_end = frame_end + else: + output_frame_start = frame_start_handle + output_frame_end = frame_end_handle + return { "fps": float(instance.data["fps"]), "frame_start": frame_start, @@ -248,21 +352,50 @@ def prepare_temp_data(self, instance): "handle_end": handle_end, "frame_start_handle": frame_start_handle, "frame_end_handle": frame_end_handle, - "output_frame_start": frame_start_handle, - "output_frame_end": frame_end_handle, + "output_frame_start": output_frame_start, + "output_frame_end": output_frame_end, "pixel_aspect": instance.data.get("pixelAspect", 1), "resolution_width": instance.data.get("resolutionWidth"), "resolution_height": instance.data.get("resolutionHeight"), + "origin_repre": repre, + "source_is_sequence": self.source_is_sequence(repre), + "without_handles": without_handles } - def input_output_paths(self, repre, output_def): - staging_dir = repre["stagingDir"] + def input_output_paths(self, new_repre, output_def, temp_data): + staging_dir = new_repre["stagingDir"] + repre = temp_data["origin_repre"] + + if temp_data["source_is_sequence"]: + collections, remainder = clique.assemble(repre["files"]) + + full_input_path = os.path.join( + staging_dir, + collections[0].format("{head}{padding}{tail}") + ) + + filename = collections[0].format("{head}") + if filename.endswith("."): + filename = filename[:-1] + else: + full_input_path = os.path.join( + staging_dir, repre["files"] + ) + filename = os.path.splitext(repre["files"])[0] + + filename_suffix = output_def["filename_suffix"] + + output_ext = output_def.get("ext") + # Use source extension if definition do not specify it + if output_ext is None: + output_ext = os.path.splitext(full_input_path)[1] # TODO Define if extension should have dot or not - output_ext = output_def.get("ext") or "mov" if output_ext.startswith("."): output_ext = output_ext[1:] + new_repre["ext"] = output_ext + self.log.debug( "New representation ext: `{}`".format(output_ext) ) @@ -276,43 +409,27 @@ def input_output_paths(self, repre, output_def): # - In that case `output_def.get("tags") or []` should replace # `repre["tags"]`. output_is_sequence = ( - "sequence" in repre["tags"] + "sequence" in new_repre["tags"] and (output_ext in self.image_exts) ) - if self.repre_has_sequence(repre): - collections, remainder = clique.assemble(repre["files"]) - - full_input_path = os.path.join( - staging_dir, - collections[0].format("{head}{padding}{tail}") - ) - - filename = collections[0].format("{head}") - if filename.endswith("."): - filename = filename[:-1] - else: - full_input_path = os.path.join( - staging_dir, repre["files"] - ) - filename = os.path.splitext(repre["files"])[0] - - filename_suffix = output_def["filename_suffix"] # QUESTION This breaks Anatomy template system is it ok? # QUESTION How do we care about multiple outputs with same - # extension? (Expect we don't...) + # extension? (Expectings are: We don't...) # - possible solution add "<{review_suffix}>" into templates # but that may cause issues when clients remove that (and it's # ugly). if output_is_sequence: - filename_base = "{}_{}".format( - filename, filename_suffix - ) - repr_file = "{}.%08d.{}".format( - filename_base, output_ext - ) + filename_base = "{}_{}".format(filename, filename_suffix) + repr_file = "{}.%08d.{}".format(filename_base, output_ext) + + new_repre_files = [] + frame_start = temp_data["output_frame_start"] + frame_end = temp_data["output_frame_end"] + for frame in range(frame_start, frame_end + 1): + new_repre_files.append(repr_file % frame) - repre["sequence_file"] = repr_file + new_repre["sequence_file"] = repr_file full_output_path = os.path.join( staging_dir, filename_base, repr_file ) @@ -322,6 +439,17 @@ def input_output_paths(self, repre, output_def): filename, filename_suffix, output_ext ) full_output_path = os.path.join(staging_dir, repr_file) + new_repre_files = repr_file + + new_repre["files"] = new_repre_files + + staging_dir = os.path.normpath(os.path.dirname(full_output_path)) + if not os.path.exists(staging_dir): + self.log.debug("Creating dir: {}".format(staging_dir)) + os.makedirs(staging_dir) + + # Set stagingDir + new_repre["stagingDir"] = staging_dir self.log.debug("Input path {}".format(full_input_path)) self.log.debug("Output path {}".format(full_output_path)) @@ -338,7 +466,7 @@ def audio_args(self, instance, temp_data): for audio in audio_inputs: # NOTE modified, always was expected "frameStartFtrack" which is - # STANGE?!!! + # STRANGE?!!! There should be different key, right? # TODO use different frame start! offset_seconds = 0 frame_start_ftrack = instance.data.get("frameStartFtrack") @@ -366,7 +494,11 @@ def audio_args(self, instance, temp_data): return audio_in_args, audio_filters, audio_out_args - def resolution_ratios(self, temp_data, output_def, repre): + def resolution_ratios(self, temp_data, output_def, new_repre): + # TODO This is not implemented and requires reimplementation since + # self.to_width and self.to_height are not set. + + # TODO get width, height from source output_width = output_def.get("width") output_height = output_def.get("height") output_pixel_aspect = output_def.get("aspect_ratio") @@ -394,19 +526,18 @@ def resolution_ratios(self, temp_data, output_def, repre): float(self.to_height) / (resolution_height * pixel_aspect) ) - self.log.debug("__ scale_factor: `{}`".format(scale_factor)) + self.log.debug("scale_factor: `{}`".format(scale_factor)) filters = [] # letter_box if output_letterbox: ffmpeg_width = self.to_width ffmpeg_height = self.to_height - if "reformat" not in repre["tags"]: + if "reformat" not in new_repre["tags"]: output_letterbox /= pixel_aspect if resolution_ratio_test != delivery_ratio_test: ffmpeg_width = resolution_width - ffmpeg_height = int( - resolution_height * pixel_aspect) + ffmpeg_height = int(resolution_height * pixel_aspect) else: if resolution_ratio_test != delivery_ratio_test: output_letterbox /= scale_factor @@ -452,7 +583,7 @@ def resolution_ratios(self, temp_data, output_def, repre): / (float(resolution_width) * pixel_aspect) ) self.log.debug( - "__ scale_factor: `{}`".format(scale_factor) + "scale_factor: `{}`".format(scale_factor) ) height_scale = int(resolution_height * scale_factor) height_half_pad = int((self.to_height - height_scale) / 2) @@ -467,12 +598,40 @@ def resolution_ratios(self, temp_data, output_def, repre): ) filters.append( "pad={}:{}:{}:{}:black".format( - self.to_width, self.to_height, + self.to_width, + self.to_height, width_half_pad, height_half_pad + ) ) filters.append("setsar=1") + new_repre["resolutionHeight"] = resolution_height + new_repre["resolutionWidth"] = resolution_width + + return filters + + def lut_filters(self, new_repre, instance, input_args): + filters = [] + # baking lut file application + lut_path = instance.data.get("lutPath") + if not lut_path or "bake-lut" not in new_repre["tags"]: + return filters + + # Prepare path for ffmpeg argument + lut_path = lut_path.replace("\\", "/").replace(":", "\\:") + + # Remove gamma from input arguments + if "-gamma" in input_args: + input_args.remove("-gamme") + + # Prepare filters + filters.append("lut3d=file='{}'".format(lut_path)) + # QUESTION hardcoded colormatrix? + filters.append("colormatrix=bt601:bt709") + + self.log.info("Added Lut to ffmpeg command") + return filters def main_family_from_instance(self, instance): @@ -556,7 +715,6 @@ def find_matching_profile(self, profiles, filter_data): matching_profiles = None highest_profile_points = -1 - profile_values = {} # Each profile get 1 point for each matching filter. Profile with most # points is returnd. For cases when more than one profile will match # are also stored ordered lists of matching values. From ed84cf293f9fc4fb8587672da707a44d92889e46 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 23 Apr 2020 15:41:11 +0200 Subject: [PATCH 11/71] fixed typo --- pype/plugins/global/publish/extract_review.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 170193ff122..e2814d8eafb 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -54,7 +54,7 @@ def process(self, instance): # Make sure cleanup happens and pop representations with "delete" tag. for repre in tuple(instance.data["representations"]): tags = repre.get("tags") or [] - if "delete" if tags: + if "delete" in tags: instance.data["representations"].remove(repre) def main_process(self, instance): From 2e85ef792a470d8bb29538d1a2b706ab197b6806 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 11:28:03 +0200 Subject: [PATCH 12/71] added ffrpobe streams getting --- pype/plugins/global/publish/extract_review.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index e2814d8eafb..242da397b25 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1,6 +1,8 @@ import os import re import copy +import json +import subprocess import pyblish.api import clique import pype.api @@ -32,7 +34,8 @@ class ExtractReview(pyblish.api.InstancePlugin): supported_exts = image_exts + video_exts # Path to ffmpeg - path_to_ffmpeg = None + ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + ffprobe_path = pype.lib.get_ffmpeg_tool_path("ffprobe") # Preset attributes profiles = None @@ -83,9 +86,6 @@ def main_process(self, instance): definition["filename_suffix"] = filename_suffix profile_outputs.append(definition) - if self.path_to_ffmpeg is None: - self.path_to_ffmpeg = pype.lib.get_ffmpeg_tool_path("ffmpeg") - # Loop through representations for repre in tuple(instance.data["representations"]): tags = repre.get("tags", []) @@ -302,7 +302,7 @@ def ffmpeg_full_args( audio_filters.append(arg) all_args = [] - all_args.append(self.path_to_ffmpeg) + all_args.append(self.ffmpeg_path) all_args.extend(input_args) if video_filters: all_args.append("-filter:v {}".format(",".join(video_filters))) @@ -634,6 +634,20 @@ def lut_filters(self, new_repre, instance, input_args): return filters + def ffprobe_streams(self, path_to_file): + args = [ + self.ffprobe_path, + "-v quiet", + "-print_format json", + "-show_format", + "-show_streams" + "\"{}\"".format(path_to_file) + ] + command = " ".join(args) + popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + + return json.loads(popen.communicate()[0])["streams"][0] + def main_family_from_instance(self, instance): family = instance.data.get("family") if not family: From a8a2efa975218da0a0d90e434e0aeab1016399da Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 15:28:33 +0200 Subject: [PATCH 13/71] extract should work now --- pype/plugins/global/publish/extract_review.py | 186 ++++++++++-------- 1 file changed, 103 insertions(+), 83 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 242da397b25..04c987f7178 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -451,6 +451,10 @@ def input_output_paths(self, new_repre, output_def, temp_data): # Set stagingDir new_repre["stagingDir"] = staging_dir + # Store paths to temp data + temp_data["full_input_path"] = full_input_path + temp_data["full_output_path"] = full_output_path + self.log.debug("Input path {}".format(full_input_path)) self.log.debug("Output path {}".format(full_output_path)) @@ -495,119 +499,135 @@ def audio_args(self, instance, temp_data): return audio_in_args, audio_filters, audio_out_args def resolution_ratios(self, temp_data, output_def, new_repre): - # TODO This is not implemented and requires reimplementation since - # self.to_width and self.to_height are not set. + filters = [] + + letter_box = output_def.get("letter_box") + # Skip processing if both conditions are not met + if "reformat" not in new_repre["tags"] and not letter_box: + return filters + + # Get instance data + pixel_aspect = temp_data["pixel_aspect"] + input_width = temp_data["resolution_width"] + input_height = temp_data["resolution_height"] + + # If instance miss resolution settings. + if input_width is None or input_height is None: + # Use input resolution + # QUESTION Should we skip instance data and use these values + # by default? + input_data = self.ffprobe_streams(temp_data["full_input_path"]) + input_width = input_data["width"] + input_height = input_data["height"] + + self.log.debug("pixel_aspect: `{}`".format(pixel_aspect)) + self.log.debug("input_width: `{}`".format(input_width)) + self.log.debug("resolution_height: `{}`".format(input_height)) - # TODO get width, height from source + # NOTE Setting only one of `width` or `heigth` is not allowed output_width = output_def.get("width") output_height = output_def.get("height") - output_pixel_aspect = output_def.get("aspect_ratio") - output_letterbox = output_def.get("letter_box") + # Use instance resolution if output definition has not set it. + if output_width is None or output_height is None: + output_width = input_width + output_height = input_height # defining image ratios - resolution_ratio = ( - (float(resolution_width) * pixel_aspect) / resolution_height + input_res_ratio = ( + (float(input_width) * pixel_aspect) / input_height ) - delivery_ratio = float(self.to_width) / float(self.to_height) - self.log.debug("resolution_ratio: `{}`".format(resolution_ratio)) - self.log.debug("delivery_ratio: `{}`".format(delivery_ratio)) + output_res_ratio = float(output_width) / float(output_height) + self.log.debug("resolution_ratio: `{}`".format(input_res_ratio)) + self.log.debug("output_res_ratio: `{}`".format(output_res_ratio)) - # shorten two decimals long float number for testing conditions - resolution_ratio_test = float("{:0.2f}".format(resolution_ratio)) - delivery_ratio_test = float("{:0.2f}".format(delivery_ratio)) + # Round ratios to 2 decimal places for comparing + input_res_ratio = round(input_res_ratio, 2) + output_res_ratio = round(output_res_ratio, 2) # get scale factor - if resolution_ratio_test < delivery_ratio_test: - scale_factor = ( - float(self.to_width) / (resolution_width * pixel_aspect) - ) - else: - scale_factor = ( - float(self.to_height) / (resolution_height * pixel_aspect) - ) + scale_factor_by_width = ( + float(output_width) / (input_width * pixel_aspect) + ) + scale_factor_by_height = ( + float(output_height) / (input_height * pixel_aspect) + ) - self.log.debug("scale_factor: `{}`".format(scale_factor)) + self.log.debug( + "scale_factor_by_with: `{}`".format(scale_factor_by_width) + ) + self.log.debug( + "scale_factor_by_height: `{}`".format(scale_factor_by_height) + ) - filters = [] # letter_box - if output_letterbox: - ffmpeg_width = self.to_width - ffmpeg_height = self.to_height - if "reformat" not in new_repre["tags"]: - output_letterbox /= pixel_aspect - if resolution_ratio_test != delivery_ratio_test: - ffmpeg_width = resolution_width - ffmpeg_height = int(resolution_height * pixel_aspect) - else: - if resolution_ratio_test != delivery_ratio_test: - output_letterbox /= scale_factor + letter_box = output_def.get("letter_box") + if letter_box: + ffmpeg_width = output_width + ffmpeg_height = output_height + if "reformat" in new_repre["tags"]: + if input_res_ratio == output_res_ratio: + letter_box /= pixel_aspect + elif input_res_ratio < output_res_ratio: + letter_box /= scale_factor_by_width else: - output_letterbox /= pixel_aspect - - filters.append( - "scale={}x{}:flags=lanczos".format(ffmpeg_width, ffmpeg_height) - ) - # QUESTION shouldn't this contain aspect ration value instead of 1? - filters.append( - "setsar=1" + letter_box /= scale_factor_by_height + else: + letter_box /= pixel_aspect + if input_res_ratio != output_res_ratio: + ffmpeg_width = input_width + ffmpeg_height = int(input_height * pixel_aspect) + + # QUESTION Is scale required when ffmpeg_width is same as + # output_width and ffmpeg_height as output_height + scale_filter = "scale={0}x{1}:flags=lanczos".format( + ffmpeg_width, ffmpeg_height ) - filters.append(( - "drawbox=0:0:iw:round((ih-(iw*(1/{})))/2):t=fill:c=black" - ).format(output_letterbox)) - filters.append(( + top_box = ( + "drawbox=0:0:iw:round((ih-(iw*(1/{0})))/2):t=fill:c=black" + ).format(letter_box) + + bottom_box = ( "drawbox=0:ih-round((ih-(iw*(1/{0})))/2)" ":iw:round((ih-(iw*(1/{0})))/2):t=fill:c=black" - ).format(output_letterbox)) + ).format(letter_box) - self.log.debug("pixel_aspect: `{}`".format(pixel_aspect)) - self.log.debug("resolution_width: `{}`".format(resolution_width)) - self.log.debug("resolution_height: `{}`".format(resolution_height)) + # Add letter box filters + filters.extend([scale_filter, "setsar=1", top_box, bottom_box]) # scaling none square pixels and 1920 width - # QUESTION: again check only output tags or repre tags - # WARNING: Duplication of filters when letter_box is set (or not?) - if "reformat" in repre["tags"]: - if resolution_ratio_test < delivery_ratio_test: - self.log.debug("lower then delivery") - width_scale = int(self.to_width * scale_factor) - width_half_pad = int((self.to_width - width_scale) / 2) - height_scale = self.to_height + if "reformat" in new_repre["tags"]: + if input_res_ratio < output_res_ratio: + self.log.debug("lower then output") + width_scale = int(output_width * scale_factor_by_width) + width_half_pad = int((output_width - width_scale) / 2) + height_scale = output_height height_half_pad = 0 else: - self.log.debug("heigher then delivery") - width_scale = self.to_width + self.log.debug("heigher then output") + width_scale = output_width width_half_pad = 0 - scale_factor = ( - float(self.to_width) - / (float(resolution_width) * pixel_aspect) - ) - self.log.debug( - "scale_factor: `{}`".format(scale_factor) - ) - height_scale = int(resolution_height * scale_factor) - height_half_pad = int((self.to_height - height_scale) / 2) + height_scale = int(input_height * scale_factor_by_width) + height_half_pad = int((output_height - height_scale) / 2) self.log.debug("width_scale: `{}`".format(width_scale)) self.log.debug("width_half_pad: `{}`".format(width_half_pad)) self.log.debug("height_scale: `{}`".format(height_scale)) self.log.debug("height_half_pad: `{}`".format(height_half_pad)) - filters.append( - "scale={}x{}:flags=lanczos".format(width_scale, height_scale) - ) - filters.append( - "pad={}:{}:{}:{}:black".format( - self.to_width, - self.to_height, - width_half_pad, - height_half_pad - ) - ) - filters.append("setsar=1") + filters.extend([ + "scale={0}x{1}:flags=lanczos".format( + width_scale, height_scale + ), + "pad={0}:{1}:{2}:{3}:black".format( + output_width, output_height, + width_half_pad, height_half_pad + ), + "setsar=1" + ]) - new_repre["resolutionHeight"] = resolution_height - new_repre["resolutionWidth"] = resolution_width + new_repre["resolutionWidth"] = output_width + new_repre["resolutionHeight"] = output_height return filters From b011caa6086f865ca8e136b451e01934b473c107 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 18:29:07 +0200 Subject: [PATCH 14/71] few final touches --- pype/plugins/global/publish/extract_review.py | 334 ++++++++++++------ 1 file changed, 222 insertions(+), 112 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 04c987f7178..c77368bccf6 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -19,8 +19,7 @@ class ExtractReview(pyblish.api.InstancePlugin): All new represetnations are created and encoded by ffmpeg following presets found in `pype-config/presets/plugins/global/ - publish.json:ExtractReview:outputs`. To change the file extension - filter values use preset's attributes `ext_filter` + publish.json:ExtractReview:outputs`. """ label = "Extract Review" @@ -33,7 +32,7 @@ class ExtractReview(pyblish.api.InstancePlugin): video_exts = ["mov", "mp4"] supported_exts = image_exts + video_exts - # Path to ffmpeg + # FFmpeg tools paths ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") ffprobe_path = pype.lib.get_ffmpeg_tool_path("ffprobe") @@ -71,13 +70,23 @@ def main_process(self, instance): self.profiles, profile_filter_data ) if not profile: + self.log.info(( + "Skipped instance. None of profiles in presets are for" + " Host: \"{host}\" | Family: \"{family}\" | Task \"{task}\"" + ).format(**profile_filter_data)) return + self.log.debug("Matching profile: \"{}\"".format(json.dumps(profile))) + instance_families = self.families_from_instance(instance) _profile_outputs = self.filter_outputs_by_families( profile, instance_families ) if not _profile_outputs: + self.log.info(( + "Skipped instance. All output definitions from selected" + " profile does not match to instance families. \"{}\"" + ).format(str(instance_families))) return # Store `filename_suffix` to save arguments @@ -88,7 +97,7 @@ def main_process(self, instance): # Loop through representations for repre in tuple(instance.data["representations"]): - tags = repre.get("tags", []) + tags = repre.get("tags") or [] if ( "review" not in tags or "multipartExr" in tags @@ -96,35 +105,42 @@ def main_process(self, instance): ): continue - source_ext = repre["ext"] - if source_ext.startswith("."): - source_ext = source_ext[1:] + input_ext = repre["ext"] + if input_ext.startswith("."): + input_ext = input_ext[1:] - if source_ext not in self.supported_exts: + if input_ext not in self.supported_exts: + self.log.info( + "Representation has unsupported extension \"{}\"".format( + input_ext + ) + ) continue # Filter output definition by representation tags (optional) outputs = self.filter_outputs_by_tags(profile_outputs, tags) if not outputs: + self.log.info(( + "Skipped representation. All output definitions from" + " selected profile does not match to representation's" + " tags. \"{}\"" + ).format(str(tags))) continue - # Prepare instance data. - # NOTE Till this point it is not required to have set most - # of keys in instance data. So publishing won't crash if plugin - # won't get here and instance miss required keys. for output_def in outputs: + # Make sure output definition has "tags" key + if "tags" not in output_def: + output_def["tags"] = [] + # Create copy of representation new_repre = copy.deepcopy(repre) - additional_tags = output_def.get("tags") or [] - # TODO new method? - # `self.prepare_new_repre_tags(new_repre, additional_tags)` # Remove "delete" tag from new repre if there is if "delete" in new_repre["tags"]: new_repre["tags"].remove("delete") # Add additional tags from output definition to representation - for tag in additional_tags: + for tag in output_def["tags"]: if tag not in new_repre["tags"]: new_repre["tags"].append(tag) @@ -132,13 +148,13 @@ def main_process(self, instance): "New representation tags: `{}`".format(new_repre["tags"]) ) - # QUESTION Why the hell we do this, adding tags to families? - # add families - for tag in additional_tags: - if tag not in instance.data["families"]: - instance.data["families"].append(tag) + # # QUESTION Why the hell we were adding tags to families? + # # add families + # for tag in output_def["tags"]: + # if tag not in instance.data["families"]: + # instance.data["families"].append(tag) - temp_data = self.prepare_temp_data(instance, repre, new_repre) + temp_data = self.prepare_temp_data(instance, repre, output_def) ffmpeg_args = self._ffmpeg_arguments( output_def, instance, temp_data @@ -170,7 +186,8 @@ def main_process(self, instance): self.log.debug("Adding: {}".format(new_repre)) instance.data["representations"].append(new_repre) - def source_is_sequence(self, repre): + def input_is_sequence(self, repre): + """Deduce from representation data if input is sequence.""" # TODO GLOBAL ISSUE - Find better way how to find out if input # is sequence. Issues( in theory): # - there may be multiple files ant not be sequence @@ -178,8 +195,82 @@ def source_is_sequence(self, repre): # - there can be more than one collection return isinstance(repre["files"], (list, tuple)) + def prepare_temp_data(self, instance, repre, output_def): + """Prepare dictionary with values used across extractor's process. + + All data are collected from instance, context, origin representation + and output definition. + + There are few required keys in Instance data: "frameStart", "frameEnd" + and "fps". + + Args: + instance (Instance): Currently processed instance. + repre (dict): Representation from which new representation was + copied. + output_def (dict): Definition of output of this plugin. + + Returns: + dict: All data which are used across methods during process. + Their values should not change during process but new keys + with values may be added. + """ + + frame_start = instance.data["frameStart"] + handle_start = instance.data.get( + "handleStart", + instance.context.data["handleStart"] + ) + frame_end = instance.data["frameEnd"] + handle_end = instance.data.get( + "handleEnd", + instance.context.data["handleEnd"] + ) + + frame_start_handle = frame_start - handle_start + frame_end_handle = frame_end + handle_end + + # Change output frames when output should be without handles + without_handles = "no-handles" in output_def["tags"] + if without_handles: + output_frame_start = frame_start + output_frame_end = frame_end + else: + output_frame_start = frame_start_handle + output_frame_end = frame_end_handle + + return { + "fps": float(instance.data["fps"]), + "frame_start": frame_start, + "frame_end": frame_end, + "handle_start": handle_start, + "handle_end": handle_end, + "frame_start_handle": frame_start_handle, + "frame_end_handle": frame_end_handle, + "output_frame_start": output_frame_start, + "output_frame_end": output_frame_end, + "pixel_aspect": instance.data.get("pixelAspect", 1), + "resolution_width": instance.data.get("resolutionWidth"), + "resolution_height": instance.data.get("resolutionHeight"), + "origin_repre": repre, + "input_is_sequence": self.input_is_sequence(repre), + "without_handles": without_handles + } + def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): - # TODO this may hold class which may be easier to work with + """Prepares ffmpeg arguments for expected extraction. + + Prepares input and output arguments based on output definition and + input files. + + Args: + output_def (dict): Currently processed output definition. + instance (Instance): Currently processed instance. + new_repre (dict): Reprensetation representing output of this + process. + temp_data (dict): Base data for successfull process. + """ + # Get FFmpeg arguments from profile presets out_def_ffmpeg_args = output_def.get("ffmpeg_args") or {} @@ -200,15 +291,13 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): ) ffmpeg_output_args.append("-frames:v {}".format(duration_frames)) - if temp_data["source_is_sequence"]: - # NOTE removed "detectedStart" key handling (NOT SET) - + if temp_data["input_is_sequence"]: # Set start frame ffmpeg_input_args.append( "-start_number {}".format(temp_data["output_frame_start"]) ) - # TODO add fps mapping `{fps: fraction}` + # TODO add fps mapping `{fps: fraction}` ? # - e.g.: { # "25": "25/1", # "24": "24/1", @@ -221,7 +310,7 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): elif temp_data["without_handles"]: # QUESTION Shall we change this to use filter: - # `select="gte(n\,handle_start),setpts=PTS-STARTPTS` + # `select="gte(n\,{handle_start}),setpts=PTS-STARTPTS` # Pros: # 1.) Python is not good at float operation # 2.) FPS on instance may not be same as input's @@ -241,12 +330,12 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): ffmpeg_audio_filters.extend(audio_filters) ffmpeg_output_args.extend(audio_out_args) - # In case audio is longer than video`. # QUESTION what if audio is shoter than video? + # In case audio is longer than video`. if "-shortest" not in ffmpeg_output_args: ffmpeg_output_args.append("-shortest") - res_filters = self.resolution_ratios(temp_data, output_def, new_repre) + res_filters = self.rescaling_filters(temp_data, output_def, new_repre) ffmpeg_video_filters.extend(res_filters) ffmpeg_input_args = self.split_ffmpeg_args(ffmpeg_input_args) @@ -254,7 +343,7 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): lut_filters = self.lut_filters(new_repre, instance, ffmpeg_input_args) ffmpeg_video_filters.extend(lut_filters) - # WARNING This must be latest added item to output arguments. + # NOTE This must be latest added item to output arguments. ffmpeg_output_args.append("\"{}\"".format(full_output_path)) return self.ffmpeg_full_args( @@ -265,6 +354,11 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): ) def split_ffmpeg_args(self, in_args): + """Makes sure all entered arguments are separated in individual items. + + Split each argument string with " -" to identify if string contains + one or more arguments. + """ splitted_args = [] for arg in in_args: sub_args = arg.split(" -") @@ -284,6 +378,22 @@ def split_ffmpeg_args(self, in_args): def ffmpeg_full_args( self, input_args, video_filters, audio_filters, output_args ): + """Post processing of collected FFmpeg arguments. + + Just verify that output arguments does not contain video or audio + filters which may cause issues because of duplicated argument entry. + Filters found in output arguments are moved to list they belong to. + + Args: + input_args (list): All collected ffmpeg arguments with inputs. + video_filters (list): All collected video filters. + audio_filters (list): All collected audio filters. + output_args (list): All collected ffmpeg output arguments with + output filepath. + + Returns: + list: Containing all arguments ready to run in subprocess. + """ output_args = self.split_ffmpeg_args(output_args) video_args_dentifiers = ["-vf", "-filter:v"] @@ -314,59 +424,23 @@ def ffmpeg_full_args( return all_args - def prepare_temp_data(self, instance, repre, new_repre): - frame_start = instance.data["frameStart"] - handle_start = instance.data.get( - "handleStart", - instance.context.data["handleStart"] - ) - frame_end = instance.data["frameEnd"] - handle_end = instance.data.get( - "handleEnd", - instance.context.data["handleEnd"] - ) + def input_output_paths(self, new_repre, output_def, temp_data): + """Deduce input nad output file paths based on entered data. - frame_start_handle = frame_start - handle_start - frame_end_handle = frame_end + handle_end + Input may be sequence of images, video file or single image file and + same can be said about output, this method helps to find out what + their paths are. - # NOTE used different key for final frame start/end to not confuse - # those who don't know what - # - e.g. "frame_start_output" - # QUESTION should we use tags ONLY from output definition? - # - In that case `output_def.get("tags") or []` should replace - # `repre["tags"]`. - # Change output frames when output should be without handles - without_handles = "no-handles" in new_repre["tags"] - if without_handles: - output_frame_start = frame_start - output_frame_end = frame_end - else: - output_frame_start = frame_start_handle - output_frame_end = frame_end_handle + It is validated that output directory exist and creates if not. - return { - "fps": float(instance.data["fps"]), - "frame_start": frame_start, - "frame_end": frame_end, - "handle_start": handle_start, - "handle_end": handle_end, - "frame_start_handle": frame_start_handle, - "frame_end_handle": frame_end_handle, - "output_frame_start": output_frame_start, - "output_frame_end": output_frame_end, - "pixel_aspect": instance.data.get("pixelAspect", 1), - "resolution_width": instance.data.get("resolutionWidth"), - "resolution_height": instance.data.get("resolutionHeight"), - "origin_repre": repre, - "source_is_sequence": self.source_is_sequence(repre), - "without_handles": without_handles - } + During process are set "files", "stagingDir", "ext" and + "sequence_file" (if output is sequence) keys to new representation. + """ - def input_output_paths(self, new_repre, output_def, temp_data): staging_dir = new_repre["stagingDir"] repre = temp_data["origin_repre"] - if temp_data["source_is_sequence"]: + if temp_data["input_is_sequence"]: collections, remainder = clique.assemble(repre["files"]) full_input_path = os.path.join( @@ -386,7 +460,7 @@ def input_output_paths(self, new_repre, output_def, temp_data): filename_suffix = output_def["filename_suffix"] output_ext = output_def.get("ext") - # Use source extension if definition do not specify it + # Use input extension if output definition do not specify it if output_ext is None: output_ext = os.path.splitext(full_input_path)[1] @@ -394,38 +468,29 @@ def input_output_paths(self, new_repre, output_def, temp_data): if output_ext.startswith("."): output_ext = output_ext[1:] + # Store extension to representation new_repre["ext"] = output_ext - self.log.debug( - "New representation ext: `{}`".format(output_ext) - ) + self.log.debug("New representation ext: `{}`".format(output_ext)) # Output is image file sequence witht frames - # QUESTION Shall we do it in opposite? Expect that if output - # extension is image format and input is sequence or video - # format then do sequence and single frame only if tag is - # "single-frame" (or similar) - # QUESTION should we use tags ONLY from output definition? - # - In that case `output_def.get("tags") or []` should replace - # `repre["tags"]`. output_is_sequence = ( - "sequence" in new_repre["tags"] - and (output_ext in self.image_exts) + (output_ext in self.image_exts) + and "sequence" in output_def["tags"] ) - # QUESTION This breaks Anatomy template system is it ok? - # QUESTION How do we care about multiple outputs with same - # extension? (Expectings are: We don't...) - # - possible solution add "<{review_suffix}>" into templates - # but that may cause issues when clients remove that (and it's - # ugly). if output_is_sequence: - filename_base = "{}_{}".format(filename, filename_suffix) - repr_file = "{}.%08d.{}".format(filename_base, output_ext) - new_repre_files = [] frame_start = temp_data["output_frame_start"] frame_end = temp_data["output_frame_end"] + + filename_base = "{}_{}".format(filename, filename_suffix) + # Temporary tempalte for frame filling. Example output: + # "basename.%04d.mov" when `frame_end` == 1001 + repr_file = "{}.%{:0>2}d.{}".format( + filename_base, len(str(frame_end)), output_ext + ) + for frame in range(frame_start, frame_end + 1): new_repre_files.append(repr_file % frame) @@ -441,14 +506,16 @@ def input_output_paths(self, new_repre, output_def, temp_data): full_output_path = os.path.join(staging_dir, repr_file) new_repre_files = repr_file + # Store files to representation new_repre["files"] = new_repre_files + # Make sure stagingDire exists staging_dir = os.path.normpath(os.path.dirname(full_output_path)) if not os.path.exists(staging_dir): self.log.debug("Creating dir: {}".format(staging_dir)) os.makedirs(staging_dir) - # Set stagingDir + # Store stagingDir to representaion new_repre["stagingDir"] = staging_dir # Store paths to temp data @@ -461,6 +528,7 @@ def input_output_paths(self, new_repre, output_def, temp_data): return full_input_path, full_output_path def audio_args(self, instance, temp_data): + """Prepares FFMpeg arguments for audio inputs.""" audio_in_args = [] audio_filters = [] audio_out_args = [] @@ -498,13 +566,18 @@ def audio_args(self, instance, temp_data): return audio_in_args, audio_filters, audio_out_args - def resolution_ratios(self, temp_data, output_def, new_repre): + def rescaling_filters(self, temp_data, output_def, new_repre): + """Prepare vieo filters based on tags in new representation. + + It is possible to add letterboxes to output video or rescale to + different resolution. + + During this preparation "resolutionWidth" and "resolutionHeight" are + set to new representation. + """ filters = [] letter_box = output_def.get("letter_box") - # Skip processing if both conditions are not met - if "reformat" not in new_repre["tags"] and not letter_box: - return filters # Get instance data pixel_aspect = temp_data["pixel_aspect"] @@ -513,9 +586,9 @@ def resolution_ratios(self, temp_data, output_def, new_repre): # If instance miss resolution settings. if input_width is None or input_height is None: - # Use input resolution - # QUESTION Should we skip instance data and use these values + # QUESTION Shall we skip instance data and use these values # by default? + # Use input resolution input_data = self.ffprobe_streams(temp_data["full_input_path"]) input_width = input_data["width"] input_height = input_data["height"] @@ -524,6 +597,12 @@ def resolution_ratios(self, temp_data, output_def, new_repre): self.log.debug("input_width: `{}`".format(input_width)) self.log.debug("resolution_height: `{}`".format(input_height)) + # Skip processing if both conditions are not met + if "reformat" not in new_repre["tags"] and not letter_box: + new_repre["resolutionWidth"] = input_width + new_repre["resolutionHeight"] = input_height + return filters + # NOTE Setting only one of `width` or `heigth` is not allowed output_width = output_def.get("width") output_height = output_def.get("height") @@ -579,12 +658,12 @@ def resolution_ratios(self, temp_data, output_def, new_repre): # QUESTION Is scale required when ffmpeg_width is same as # output_width and ffmpeg_height as output_height - scale_filter = "scale={0}x{1}:flags=lanczos".format( + scale_filter = "scale={}x{}:flags=lanczos".format( ffmpeg_width, ffmpeg_height ) top_box = ( - "drawbox=0:0:iw:round((ih-(iw*(1/{0})))/2):t=fill:c=black" + "drawbox=0:0:iw:round((ih-(iw*(1/{})))/2):t=fill:c=black" ).format(letter_box) bottom_box = ( @@ -616,10 +695,10 @@ def resolution_ratios(self, temp_data, output_def, new_repre): self.log.debug("height_half_pad: `{}`".format(height_half_pad)) filters.extend([ - "scale={0}x{1}:flags=lanczos".format( + "scale={}x{}:flags=lanczos".format( width_scale, height_scale ), - "pad={0}:{1}:{2}:{3}:black".format( + "pad={}:{}:{}:{}:black".format( output_width, output_height, width_half_pad, height_half_pad ), @@ -632,6 +711,7 @@ def resolution_ratios(self, temp_data, output_def, new_repre): return filters def lut_filters(self, new_repre, instance, input_args): + """Add lut file to output ffmpeg filters.""" filters = [] # baking lut file application lut_path = instance.data.get("lutPath") @@ -650,11 +730,12 @@ def lut_filters(self, new_repre, instance, input_args): # QUESTION hardcoded colormatrix? filters.append("colormatrix=bt601:bt709") - self.log.info("Added Lut to ffmpeg command") + self.log.info("Added Lut to ffmpeg command.") return filters def ffprobe_streams(self, path_to_file): + """Load streams from entered filepath.""" args = [ self.ffprobe_path, "-v quiet", @@ -669,12 +750,14 @@ def ffprobe_streams(self, path_to_file): return json.loads(popen.communicate()[0])["streams"][0] def main_family_from_instance(self, instance): + """Returns main family of entered instance.""" family = instance.data.get("family") if not family: family = instance.data["families"][0] return family def families_from_instance(self, instance): + """Returns all families of entered instance.""" families = [] family = instance.data.get("family") if family: @@ -686,6 +769,7 @@ def families_from_instance(self, instance): return families def compile_list_of_regexes(self, in_list): + """Convert strings in entered list to compiled regex objects.""" regexes = [] if not in_list: return regexes @@ -851,6 +935,10 @@ def find_matching_profile(self, profiles, filter_data): return final_profile def families_filter_validation(self, families, output_families_filter): + """Determines if entered families intersect with families filters. + + All family values are lowered to avoid unexpected results. + """ if not output_families_filter: return True @@ -885,6 +973,17 @@ def families_filter_validation(self, families, output_families_filter): return False def filter_outputs_by_families(self, profile, families): + """Filter outputs that are not supported for instance families. + + Output definitions without families filter are marked as valid. + + Args: + profile (dict): Profile from presets matching current context. + families (list): All families of current instance. + + Returns: + list: Containg all output definitions matching entered families. + """ outputs = profile.get("outputs") or [] if not outputs: return outputs @@ -910,6 +1009,17 @@ def filter_outputs_by_families(self, profile, families): return filtered_outputs def filter_outputs_by_tags(self, outputs, tags): + """Filter output definitions by entered representation tags. + + Output definitions without tags filter are marked as valid. + + Args: + outputs (list): Contain list of output definitions from presets. + tags (list): Tags of processed representation. + + Returns: + list: Containg all output definitions matching entered tags. + """ filtered_outputs = [] repre_tags_low = [tag.lower() for tag in tags] for output_def in outputs: From 673e531689ad32b1377c11740d038f6ac457d20a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 19:03:01 +0200 Subject: [PATCH 15/71] added profile_exclusion --- pype/plugins/global/publish/extract_review.py | 102 +++++++++++------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index c77368bccf6..fe8946114d1 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -811,6 +811,67 @@ def validate_value_by_regexes(self, value, in_list): break return output + def profile_exclusion(self, matching_profiles): + """Find out most matching profile byt host, task and family match. + + Profiles are selectivelly filtered. Each profile should have + "__value__" key with list of booleans. Each boolean represents + existence of filter for specific key (host, taks, family). + Profiles are looped in sequence. In each sequence are split into + true_list and false_list. For next sequence loop are used profiles in + true_list if there are any profiles else false_list is used. + + Filtering ends when only one profile left in true_list. Or when all + existence booleans loops passed, in that case first profile from left + profiles is returned. + + Args: + matching_profiles (list): Profiles with same values. + + Returns: + dict: Most matching profile. + """ + self.log.info( + "Search for first most matching profile in match order:" + " Host name -> Task name -> Family." + ) + # Filter all profiles with highest points value. First filter profiles + # with matching host if there are any then filter profiles by task + # name if there are any and lastly filter by family. Else use first in + # list. + idx = 0 + final_profile = None + while True: + profiles_true = [] + profiles_false = [] + for profile in matching_profiles: + value = profile["__value__"] + # Just use first profile when idx is greater than values. + if not idx < len(value): + final_profile = profile + break + + if value[idx]: + profiles_true.append(profile) + else: + profiles_false.append(profile) + + if final_profile is not None: + break + + if profiles_true: + matching_profiles = profiles_true + else: + matching_profiles = profiles_false + + if len(matching_profiles) == 1: + final_profile = matching_profiles[0] + break + idx += 1 + + final_profile.pop("__value__") + return final_profile + def find_matching_profile(self, profiles, filter_data): """ Filter profiles by Host name, Task name and main Family. @@ -893,46 +954,7 @@ def find_matching_profile(self, profiles, filter_data): " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" ).format(**filter_data)) - # Filter all profiles with highest points value. First filter profiles - # with matching host if there are any then filter profiles by task - # name if there are any and lastly filter by family. Else use first in - # list. - idx = 0 - final_profile = None - while True: - profiles_true = [] - profiles_false = [] - for profile in matching_profiles: - value = profile["__value__"] - # Just use first profile when idx is greater than values. - if not idx < len(value): - final_profile = profile - break - - if value[idx]: - profiles_true.append(profile) - else: - profiles_false.append(profile) - - if final_profile is not None: - break - - if profiles_true: - matching_profiles = profiles_true - else: - matching_profiles = profiles_false - - if len(matching_profiles) == 1: - final_profile = matching_profiles[0] - break - idx += 1 - - final_profile.pop("__value__") - self.log.info( - "Using first most matching profile in match order:" - " Host name -> Task name -> Family." - ) - return final_profile + return self.profile_exclusion(matching_profiles) def families_filter_validation(self, families, output_families_filter): """Determines if entered families intersect with families filters. From c55373238bb5da99b9cfdb61b8f0e083af871830 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 19:22:53 +0200 Subject: [PATCH 16/71] make sure always is accessible path to one input file --- pype/plugins/global/publish/extract_review.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index fe8946114d1..deb4eb9ab6b 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -441,7 +441,7 @@ def input_output_paths(self, new_repre, output_def, temp_data): repre = temp_data["origin_repre"] if temp_data["input_is_sequence"]: - collections, remainder = clique.assemble(repre["files"]) + collections = clique.assemble(repre["files"])[0] full_input_path = os.path.join( staging_dir, @@ -451,12 +451,21 @@ def input_output_paths(self, new_repre, output_def, temp_data): filename = collections[0].format("{head}") if filename.endswith("."): filename = filename[:-1] + + # Make sure to have full path to one input file + full_input_path_single_file = os.path.join( + staging_dir, repre["files"][0] + ) + else: full_input_path = os.path.join( staging_dir, repre["files"] ) filename = os.path.splitext(repre["files"])[0] + # Make sure to have full path to one input file + full_input_path_single_file = full_input_path + filename_suffix = output_def["filename_suffix"] output_ext = output_def.get("ext") @@ -520,6 +529,7 @@ def input_output_paths(self, new_repre, output_def, temp_data): # Store paths to temp data temp_data["full_input_path"] = full_input_path + temp_data["full_input_path_single_file"] = full_input_path_single_file temp_data["full_output_path"] = full_output_path self.log.debug("Input path {}".format(full_input_path)) From f267476d33dc7aef32f1e39cf3063e017d227bf6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 19:23:45 +0200 Subject: [PATCH 17/71] input resolution is not taken from instance data but from input source --- pype/plugins/global/publish/extract_review.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index deb4eb9ab6b..32c8a10039a 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -591,17 +591,12 @@ def rescaling_filters(self, temp_data, output_def, new_repre): # Get instance data pixel_aspect = temp_data["pixel_aspect"] - input_width = temp_data["resolution_width"] - input_height = temp_data["resolution_height"] - - # If instance miss resolution settings. - if input_width is None or input_height is None: - # QUESTION Shall we skip instance data and use these values - # by default? - # Use input resolution - input_data = self.ffprobe_streams(temp_data["full_input_path"]) - input_width = input_data["width"] - input_height = input_data["height"] + + # NOTE Skipped using instance's resolution + full_input_path_single_file = temp_data["full_input_path_single_file"] + input_data = self.ffprobe_streams(full_input_path_single_file) + input_width = input_data["width"] + input_height = input_data["height"] self.log.debug("pixel_aspect: `{}`".format(pixel_aspect)) self.log.debug("input_width: `{}`".format(input_width)) From fcc664f31a39422a82ba42d68741a6825b17824a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 19:24:16 +0200 Subject: [PATCH 18/71] use instance resolution for output before using input's resolution --- pype/plugins/global/publish/extract_review.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 32c8a10039a..55e97c17d50 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -612,6 +612,11 @@ def rescaling_filters(self, temp_data, output_def, new_repre): output_width = output_def.get("width") output_height = output_def.get("height") # Use instance resolution if output definition has not set it. + if output_width is None or output_height is None: + output_width = temp_data["resolution_width"] + output_height = temp_data["resolution_height"] + + # Use source's input resolution instance does not have set it. if output_width is None or output_height is None: output_width = input_width output_height = input_height From 446b91e56de79ba9451900785a0fd6bfa08cb724 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 24 Apr 2020 19:25:14 +0200 Subject: [PATCH 19/71] extract review slate match new extract review and count values per each representation not per instance --- .../global/publish/extract_review_slate.py | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index aaa67bde68b..e94701a312d 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -26,46 +26,54 @@ def process(self, instance): slate_path = inst_data.get("slateFrame") ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - # values are set in ExtractReview - to_width = inst_data["reviewToWidth"] - to_height = inst_data["reviewToHeight"] - - resolution_width = inst_data.get("resolutionWidth", to_width) - resolution_height = inst_data.get("resolutionHeight", to_height) pixel_aspect = inst_data.get("pixelAspect", 1) fps = inst_data.get("fps") - # defining image ratios - resolution_ratio = ((float(resolution_width) * pixel_aspect) / - resolution_height) - delivery_ratio = float(to_width) / float(to_height) - self.log.debug("__ resolution_ratio: `{}`".format(resolution_ratio)) - self.log.debug("__ delivery_ratio: `{}`".format(delivery_ratio)) + for idx, repre in enumerate(inst_data["representations"]): + self.log.debug("__ i: `{}`, repre: `{}`".format(idx, repre)) - # get scale factor - scale_factor = float(to_height) / ( - resolution_height * pixel_aspect) + p_tags = repre.get("tags", []) + if "slate-frame" not in p_tags: + continue - # shorten two decimals long float number for testing conditions - resolution_ratio_test = float( - "{:0.2f}".format(resolution_ratio)) - delivery_ratio_test = float( - "{:0.2f}".format(delivery_ratio)) + # values are set in ExtractReview + to_width = repre["resolutionWidth"] + to_height = repre["resolutionHeight"] - if resolution_ratio_test < delivery_ratio_test: - scale_factor = float(to_width) / ( - resolution_width * pixel_aspect) + # QUESTION Should we use resolution from instance and not source's? + resolution_width = inst_data.get("resolutionWidth") + if resolution_width is None: + resolution_width = to_width - self.log.debug("__ scale_factor: `{}`".format(scale_factor)) + resolution_height = inst_data.get("resolutionHeight") + if resolution_height is None: + resolution_height = to_height - for i, repre in enumerate(inst_data["representations"]): - _remove_at_end = [] - self.log.debug("__ i: `{}`, repre: `{}`".format(i, repre)) + # defining image ratios + resolution_ratio = ( + (float(resolution_width) * pixel_aspect) / resolution_height + ) + delivery_ratio = float(to_width) / float(to_height) + self.log.debug("resolution_ratio: `{}`".format(resolution_ratio)) + self.log.debug("delivery_ratio: `{}`".format(delivery_ratio)) - p_tags = repre.get("tags", []) + # get scale factor + scale_factor = float(to_height) / ( + resolution_height * pixel_aspect) - if "slate-frame" not in p_tags: - continue + # shorten two decimals long float number for testing conditions + resolution_ratio_test = float( + "{:0.2f}".format(resolution_ratio)) + delivery_ratio_test = float( + "{:0.2f}".format(delivery_ratio)) + + if resolution_ratio_test < delivery_ratio_test: + scale_factor = float(to_width) / ( + resolution_width * pixel_aspect) + + self.log.debug("__ scale_factor: `{}`".format(scale_factor)) + + _remove_at_end = [] stagingdir = repre["stagingDir"] input_file = "{0}".format(repre["files"]) @@ -87,7 +95,7 @@ def process(self, instance): # overrides output file input_args.append("-y") # preset's input data - input_args.extend(repre["_profile"].get('input', [])) + input_args.extend(repre["outputDef"].get('input', [])) input_args.append("-loop 1 -i {}".format(slate_path)) input_args.extend([ "-r {}".format(fps), @@ -95,10 +103,11 @@ def process(self, instance): ) # output args - codec_args = repre["_profile"].get('codec', []) - output_args.extend(codec_args) # preset's output data - output_args.extend(repre["_profile"].get('output', [])) + output_args.extend(repre["outputDef"].get('output', [])) + + # Codecs are copied from source for whole input + output_args.append("-codec copy") # make sure colors are correct output_args.extend([ @@ -206,10 +215,10 @@ def process(self, instance): "name": repre["name"], "tags": [x for x in repre["tags"] if x != "delete"] } - inst_data["representations"][i].update(repre_update) + inst_data["representations"][idx].update(repre_update) self.log.debug( "_ representation {}: `{}`".format( - i, inst_data["representations"][i])) + idx, inst_data["representations"][idx])) # removing temp files for f in _remove_at_end: From 1c933741eaaa26710c5c6aa455d3d6146e5e9e18 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 13:47:59 +0200 Subject: [PATCH 20/71] burnin script can handle both new and old ExtractBurnin plugin processes --- pype/scripts/otio_burnin.py | 56 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 7c940064665..4c9e0fc4d78 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -335,22 +335,23 @@ def example(input_path, output_path): def burnins_from_data( - input_path, output_path, data, codec_data=None, overwrite=True + input_path, output_path, data, + codec_data=None, options=None, burnin_values=None, overwrite=True ): - ''' - This method adds burnins to video/image file based on presets setting. + """This method adds burnins to video/image file based on presets setting. + Extension of output MUST be same as input. (mov -> mov, avi -> avi,...) - :param input_path: full path to input file where burnins should be add - :type input_path: str - :param codec_data: all codec related arguments in list - :param codec_data: list - :param output_path: full path to output file where output will be rendered - :type output_path: str - :param data: data required for burnin settings (more info below) - :type data: dict - :param overwrite: output will be overriden if already exists, defaults to True - :type overwrite: bool + Args: + input_path (str): Full path to input file where burnins should be add. + output_path (str): Full path to output file where output will be + rendered. + data (dict): Data required for burnin settings (more info below). + codec_data (list): All codec related arguments in list. + options (dict): Options for burnins. + burnin_values (dict): Contain positioned values. + overwrite (bool): Output will be overriden if already exists, + True by default. Presets must be set separately. Should be dict with 2 keys: - "options" - sets look of burnins - colors, opacity,...(more info: ModifiedBurnins doc) @@ -391,11 +392,18 @@ def burnins_from_data( "frame_start_tc": 1, "shot": "sh0010" } - ''' - presets = config.get_presets().get('tools', {}).get('burnins', {}) - options_init = presets.get('options') + """ + # Make sure `codec_data` is list + if not codec_data: + codec_data = [] - burnin = ModifiedBurnins(input_path, options_init=options_init) + # Use legacy processing when options are not set + if options is None or burnin_values is None: + presets = config.get_presets().get("tools", {}).get("burnins", {}) + options = presets.get("options") + burnin_values = presets.get("burnins") or {} + + burnin = ModifiedBurnins(input_path, options_init=options) frame_start = data.get("frame_start") frame_end = data.get("frame_end") @@ -425,7 +433,7 @@ def burnins_from_data( if source_timecode is not None: data[SOURCE_TIMECODE_KEY[1:-1]] = SOURCE_TIMECODE_KEY - for align_text, value in presets.get('burnins', {}).items(): + for align_text, value in burnin_values.items(): if not value: continue @@ -511,11 +519,13 @@ def burnins_from_data( burnin.render(output_path, args=codec_args, overwrite=overwrite, **data) -if __name__ == '__main__': +if __name__ == "__main__": in_data = json.loads(sys.argv[-1]) burnins_from_data( - in_data['input'], - in_data['output'], - in_data['burnin_data'], - in_data['codec'] + in_data["input"], + in_data["output"], + in_data["burnin_data"], + codec_data=in_data.get("codec"), + options=in_data.get("optios"), + values=in_data.get("values") ) From 74f278d507fc54623a8108eb3db043a415b963ce Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 13:49:05 +0200 Subject: [PATCH 21/71] profile filter values are not prestored to dictionary --- pype/plugins/global/publish/extract_review.py | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 55e97c17d50..8d7aec1d2e1 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -60,20 +60,18 @@ def process(self, instance): instance.data["representations"].remove(repre) def main_process(self, instance): - profile_filter_data = { - "host": pyblish.api.registered_hosts()[-1].title(), - "family": self.main_family_from_instance(instance), - "task": os.environ["AVALON_TASK"] - } + host_name = pyblish.api.registered_hosts()[-1].title() + task_name = os.environ["AVALON_TASK"] + family = self.main_family_from_instance(instance) profile = self.find_matching_profile( - self.profiles, profile_filter_data + host_name, task_name, family ) if not profile: self.log.info(( "Skipped instance. None of profiles in presets are for" " Host: \"{host}\" | Family: \"{family}\" | Task \"{task}\"" - ).format(**profile_filter_data)) + ).format(host_name, family, task_name)) return self.log.debug("Matching profile: \"{}\"".format(json.dumps(profile))) @@ -882,7 +880,7 @@ def profile_exclusion(self, matching_profiles): final_profile.pop("__value__") return final_profile - def find_matching_profile(self, profiles, filter_data): + def find_matching_profile(self, host_name, task_name, family): """ Filter profiles by Host name, Task name and main Family. Filtering keys are "hosts" (list), "tasks" (list), "families" (list). @@ -890,24 +888,24 @@ def find_matching_profile(self, profiles, filter_data): Args: profiles (list): Profiles definition from presets. - filter_data (dict): Dictionary with data for filtering. - Required keys are "host" - Host name, "task" - Task name - and "family" - Main instance family. + host_name (str): Current running host name. + task_name (str): Current context task name. + family (str): Main family of current Instance. Returns: dict/None: Return most matching profile or None if none of profiles match at least one criteria. """ - host_name = filter_data["host"] - task_name = filter_data["task"] - family = filter_data["family"] matching_profiles = None + if not self.profiles: + return matching_profiles + highest_profile_points = -1 # Each profile get 1 point for each matching filter. Profile with most # points is returnd. For cases when more than one profile will match # are also stored ordered lists of matching values. - for profile in profiles: + for profile in self.profiles: profile_points = 0 profile_value = [] @@ -950,8 +948,8 @@ def find_matching_profile(self, profiles, filter_data): if not matching_profiles: self.log.info(( "None of profiles match your setup." - " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" - ).format(**filter_data)) + " Host \"{}\" | Task: \"{}\" | Family: \"{}\"" + ).format(host_name, task_name, family)) return if len(matching_profiles) == 1: @@ -961,8 +959,8 @@ def find_matching_profile(self, profiles, filter_data): self.log.warning(( "More than one profile match your setup." - " Host \"{host}\" | Task: \"{task}\" | Family: \"{family}\"" - ).format(**filter_data)) + " Host \"{}\" | Task: \"{}\" | Family: \"{}\"" + ).format(host_name, task_name, family)) return self.profile_exclusion(matching_profiles) From fc694cd516f0326b7c978f52211bb74d029caf9b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 13:49:42 +0200 Subject: [PATCH 22/71] "output_filter" for output definition in presets was renamed to "filter" --- pype/plugins/global/publish/extract_review.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 8d7aec1d2e1..e7eb210b2df 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -999,7 +999,6 @@ def families_filter_validation(self, families, output_families_filter): if valid: return True - return False def filter_outputs_by_families(self, profile, families): @@ -1024,7 +1023,7 @@ def filter_outputs_by_families(self, profile, families): filtered_outputs = {} for filename_suffix, output_def in outputs.items(): - output_filters = output_def.get("output_filter") + output_filters = output_def.get("filter") # When filters not set then skip filtering process if not output_filters: filtered_outputs[filename_suffix] = output_def @@ -1054,7 +1053,7 @@ def filter_outputs_by_tags(self, outputs, tags): repre_tags_low = [tag.lower() for tag in tags] for output_def in outputs: valid = True - output_filters = output_def.get("output_filter") + output_filters = output_def.get("filter") if output_filters: # Check tag filters tag_filters = output_filters.get("tags") From 59ce9a5a6dc576826888b3dccce3eb4250499903 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 13:52:26 +0200 Subject: [PATCH 23/71] extract burnin updated to care about new burnin presets with backwards compatibility --- pype/plugins/global/publish/extract_burnin.py | 736 ++++++++++++++++++ 1 file changed, 736 insertions(+) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 7668eafd2a7..693940accfa 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -1,10 +1,13 @@ import os +import re import json import copy import pype.api import pyblish +StringType = type("") + class ExtractBurnin(pype.api.Extractor): """ @@ -21,10 +24,743 @@ class ExtractBurnin(pype.api.Extractor): hosts = ["nuke", "maya", "shell"] optional = True + positions = [ + "top_left", "top_centered", "top_right", + "bottom_right", "bottom_centered", "bottom_left" + ] + # Default options for burnins for cases that are not set in presets. + default_options = { + "opacity": 1, + "x_offset": 5, + "y_offset": 5, + "bg_padding": 5, + "bg_opacity": 0.5, + "font_size": 42 + } + + # Preset attributes + profiles = None + options = None + fields = None + def process(self, instance): + # QUESTION what is this for and should we raise an exception? if "representations" not in instance.data: raise RuntimeError("Burnin needs already created mov to work on.") + if self.profiles is None: + return self.legacy_process(instance) + self.main_process(instance) + + # Remove any representations tagged for deletion. + # QUESTION Is possible to have representation with "delete" tag? + for repre in tuple(instance.data["representations"]): + if "delete" in repre.get("tags", []): + self.log.debug("Removing representation: {}".format(repre)) + instance.data["representations"].remove(repre) + + self.log.debug(instance.data["representations"]) + + def main_process(self, instance): + host_name = pyblish.api.registered_hosts()[-1].title() + task_name = os.environ["AVALON_TASK"] + family = self.main_family_from_instance(instance) + + # Find profile most matching current host, task and instance family + profile = self.find_matching_profile(host_name, task_name, family) + if not profile: + self.log.info(( + "Skipped instance. None of profiles in presets are for" + " Host: \"{}\" | Family: \"{}\" | Task \"{}\"" + ).format(host_name, family, task_name)) + return + + # Pre-filter burnin definitions by instance families + burnin_defs = self.filter_burnins_by_families(profile, instance) + if not burnin_defs: + self.log.info(( + "Skipped instance. Burnin definitions are not set for profile" + " Host: \"{}\" | Family: \"{}\" | Task \"{}\" | Profile \"{}\"" + ).format(host_name, family, task_name, profile)) + return + + # Prepare burnin options + profile_options = copy.deepcopy(self.default_options) + for key, value in (self.options or {}).items(): + if value is not None: + profile_options[key] = value + + # Prepare global burnin values from presets + profile_burnins = {} + for key, value in (self.fields or {}).items(): + key_low = key.lower() + if key_low in self.positions: + if value is not None: + profile_burnins[key_low] = value + + # Prepare basic data for processing + _burnin_data, _temp_data = self.prepare_basic_data(instance) + + anatomy = instance.context.data["anatomy"] + scriptpath = self.burnin_script_path() + executable = self.python_executable_path() + + for idx, repre in tuple(instance.data["representations"].items()): + self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) + if not self.repres_is_valid(repre): + continue + + # Filter output definition by representation tags (optional) + repre_burnin_defs = self.filter_burnins_by_tags( + burnin_defs, repre["tags"] + ) + if not repre_burnin_defs: + self.log.info(( + "Skipped representation. All burnin definitions from" + " selected profile does not match to representation's" + " tags. \"{}\"" + ).format(str(repre["tags"]))) + continue + + # Create copy of `_burnin_data` and `_temp_data` for repre. + burnin_data = copy.deepcopy(_burnin_data) + temp_data = copy.deepcopy(_temp_data) + + # Prepare representation based data. + self.prepare_repre_data(instance, repre, burnin_data, temp_data) + + # Add anatomy keys to burnin_data. + filled_anatomy = anatomy.format_all(burnin_data) + burnin_data["anatomy"] = filled_anatomy.get_solved() + + files_to_delete = [] + for filename_suffix, burnin_def in repre_burnin_defs.items(): + new_repre = copy.deepcopy(repre) + + burnin_options = copy.deepcopy(profile_options) + burnin_values = copy.deepcopy(profile_burnins) + + # Options overrides + for key, value in (burnin_def.get("options") or {}).items(): + # Set or override value if is valid + if value is not None: + burnin_options[key] = value + + # Burnin values overrides + for key, value in burnin_def.items(): + key_low = key.lower() + if key_low in self.positions: + if value is not None: + # Set or override value if is valid + burnin_values[key_low] = value + + elif key_low in burnin_values: + # Pop key if value is set to None (null in json) + burnin_values.pop(key_low) + + # Remove "delete" tag from new representation + if "delete" in new_repre["tags"]: + new_repre["tags"].remove("delete") + + # Update outputName to be able have multiple outputs + # Join previous "outputName" with filename suffix + new_repre["outputName"] = "_".join( + [new_repre["outputName"], filename_suffix] + ) + + # Prepare paths and files for process. + self.input_output_paths(new_repre, temp_data, filename_suffix) + + # Data for burnin script + script_data = { + "input": temp_data["full_input_path"], + "output": temp_data["full_output_path"], + "burnin_data": burnin_data, + "options": burnin_options, + "values": burnin_values + } + + self.log.debug("script_data: {}".format(script_data)) + + # Dump data to string + dumped_script_data = json.dumps(script_data) + + # Prepare subprocess arguments + args = [executable, scriptpath, dumped_script_data] + self.log.debug("Executing: {}".format(args)) + + # Run burnin script + output = pype.api.subprocess(args) + self.log.debug("Output: {}".format(output)) + + for filepath in temp_data["full_input_paths"]: + filepath = filepath.replace("\\", "/") + if filepath not in files_to_delete: + files_to_delete.append(filepath) + + # Add new representation to instance + instance.data["representations"].append(new_repre) + + # Remove source representation + # NOTE we maybe can keep source representation if necessary + instance.data["representations"].remove(repre) + + # Delete input files + for filepath in files_to_delete: + if os.path.exists(filepath): + os.remove(filepath) + self.log.debug("Removed: \"{}\"".format(filepath)) + + def prepare_basic_data(self, instance): + """Pick data from instance for processing and for burnin strings. + + Args: + instance (Instance): Currently processed instance. + + Returns: + tuple: `(burnin_data, temp_data)` - `burnin_data` contain data for + filling burnin strings. `temp_data` are for repre pre-process + preparation. + """ + self.log.debug("Prepring basic data for burnins") + context = instance.context + + version = instance.data.get("version") + if version is None: + version = context.data.get("version") + + frame_start = instance.data.get("frameStart") + if frame_start is None: + self.log.warning( + "Key \"frameStart\" is not set. Setting to \"0\"." + ) + frame_start = 0 + frame_start = int(frame_start) + + frame_end = instance.data.get("frameEnd") + if frame_end is None: + self.log.warning( + "Key \"frameEnd\" is not set. Setting to \"1\"." + ) + frame_end = 1 + frame_end = int(frame_end) + + handles = instance.data.get("handles") + if handles is None: + handles = context.data.get("handles") + if handles is None: + handles = 0 + + handle_start = instance.data.get("handleStart") + if handle_start is None: + handle_start = context.data.get("handleStart") + if handle_start is None: + handle_start = handles + + handle_end = instance.data.get("handleEnd") + if handle_end is None: + handle_end = context.data.get("handleEnd") + if handle_end is None: + handle_end = handles + + frame_start_handle = frame_start - handle_start + frame_end_handle = frame_end + handle_end + + burnin_data = copy.deepcopy(instance.data["anatomyData"]) + + if "slate.farm" in instance.data["families"]: + frame_start_handle += 1 + + burnin_data.update({ + "version": int(version), + "comment": context.data.get("comment") or "" + }) + + intent_label = context.data.get("intent") + if intent_label and isinstance(intent_label, dict): + intent_label = intent_label.get("label") + + if intent_label: + burnin_data["intent"] = intent_label + + temp_data = { + "frame_start": frame_start, + "frame_end": frame_end, + "frame_start_handle": frame_start_handle, + "frame_end_handle": frame_end_handle + } + + self.log.debug("Basic burnin_data: {}".format(burnin_data)) + + return burnin_data, temp_data + + def repres_is_valid(self, repre): + """Validation if representaion should be processed. + + Args: + repre (dict): Representation which should be checked. + + Returns: + bool: False if can't be processed else True. + """ + + if "burnin" not in (repre.get("tags") or []): + self.log.info("Representation don't have \"burnin\" tag.") + return False + + # ffmpeg doesn't support multipart exrs + if "multipartExr" in repre["tags"]: + self.log.info("Representation contain \"multipartExr\" tag.") + return False + return True + + def filter_burnins_by_tags(self, burnin_defs, tags): + """Filter burnin definitions by entered representation tags. + + Burnin definitions without tags filter are marked as valid. + + Args: + outputs (list): Contain list of burnin definitions from presets. + tags (list): Tags of processed representation. + + Returns: + list: Containg all burnin definitions matching entered tags. + """ + filtered_outputs = [] + repre_tags_low = [tag.lower() for tag in tags] + for burnin_def in burnin_defs: + valid = True + output_filters = burnin_def.get("filter") + if output_filters: + # Check tag filters + tag_filters = output_filters.get("tags") + if tag_filters: + tag_filters_low = [tag.lower() for tag in tag_filters] + valid = False + for tag in repre_tags_low: + if tag in tag_filters_low: + valid = True + break + + if not valid: + continue + + if valid: + filtered_outputs.append(burnin_def) + + return filtered_outputs + + def input_output_paths(self, new_repre, temp_data, filename_suffix): + """Prepare input and output paths for representation. + + Store data to `temp_data` for keys "full_input_path" which is full path + to source files optionally with sequence formatting, + "full_output_path" full path to otput with optionally with sequence + formatting, "full_input_paths" list of all source files which will be + deleted when burnin script ends, "repre_files" list of output + filenames. + + Args: + new_repre (dict): Currently processed new representation. + temp_data (dict): Temp data of representation process. + filename_suffix (str): Filename suffix added to inputl filename. + + Returns: + None: This is processing method. + """ + is_sequence = "sequence" in new_repre["tags"] + if is_sequence: + input_filename = new_repre["sequence_file"] + else: + input_filename = new_repre["files"] + + filepart_start, ext = os.path.splitext(input_filename) + dir_path, basename = os.path.split(filepart_start) + + if is_sequence: + # NOTE modified to keep name when multiple dots are in name + basename_parts = basename.split(".") + frame_part = basename_parts.pop(-1) + + basename_start = ".".join(basename_parts) + filename_suffix + new_basename = ".".join(basename_start, frame_part) + output_filename = new_basename + ext + + else: + output_filename = basename + filename_suffix + ext + + if dir_path: + output_filename = os.path.join(dir_path, output_filename) + + stagingdir = new_repre["stagingDir"] + full_input_path = os.path.join( + os.path.normpath(stagingdir), input_filename + ).replace("\\", "/") + full_output_path = os.path.join( + os.path.normpath(stagingdir), output_filename + ).replace("\\", "/") + + temp_data["full_input_path"] = full_input_path + temp_data["full_output_path"] = full_output_path + + self.log.debug("full_input_path: {}".format(full_input_path)) + self.log.debug("full_output_path: {}".format(full_output_path)) + + # Prepare full paths to input files and filenames for reprensetation + full_input_paths = [] + if is_sequence: + repre_files = [] + for frame_index in range(1, temp_data["duration"] + 1): + repre_files.append(output_filename % frame_index) + full_input_paths.append(full_input_path % frame_index) + + else: + full_input_paths.append(full_input_path) + repre_files = output_filename + + temp_data["full_input_paths"] = full_input_paths + new_repre["repre_files"] = repre_files + + def prepare_repre_data(self, instance, repre, burnin_data, temp_data): + """Prepare data for representation. + + Args: + instance (Instance): Currently processed Instance. + repre (dict): Currently processed representation. + burnin_data (dict): Copy of basic burnin data based on instance + data. + temp_data (dict): Copy of basic temp data + """ + # Add representation name to burnin data + burnin_data["representation"] = repre["name"] + + # no handles switch from profile tags + if "no-handles" in repre["tags"]: + burnin_frame_start = temp_data["frame_start"] + burnin_frame_end = temp_data["frame_end"] + + else: + burnin_frame_start = temp_data["frame_start_handle"] + burnin_frame_end = temp_data["frame_end_handle"] + + burnin_duration = burnin_frame_end - burnin_frame_start + 1 + + burnin_data.update({ + "frame_start": burnin_frame_start, + "frame_end": burnin_frame_end, + "duration": burnin_duration, + }) + temp_data["duration"] = burnin_duration + + # Add values for slate frames + burnin_slate_frame_start = burnin_frame_start + + # Move frame start by 1 frame when slate is used. + if ( + "slate" in instance.data["families"] + and "slate-frame" in repre["tags"] + ): + burnin_slate_frame_start -= 1 + + self.log.debug("burnin_slate_frame_start: {}".format( + burnin_slate_frame_start + )) + + burnin_data.update({ + "slate_frame_start": burnin_slate_frame_start, + "slate_frame_end": burnin_frame_end, + "slate_duration": ( + burnin_frame_end - burnin_slate_frame_start + 1 + ) + }) + + def find_matching_profile(self, host_name, task_name, family): + """ Filter profiles by Host name, Task name and main Family. + + Filtering keys are "hosts" (list), "tasks" (list), "families" (list). + If key is not find or is empty than it's expected to match. + + Args: + profiles (list): Profiles definition from presets. + host_name (str): Current running host name. + task_name (str): Current context task name. + family (str): Main family of current Instance. + + Returns: + dict/None: Return most matching profile or None if none of profiles + match at least one criteria. + """ + + matching_profiles = None + highest_points = -1 + for profile in self.profiles or tuple(): + profile_points = 0 + profile_value = [] + + # Host filtering + host_names = profile.get("hosts") + match = self.validate_value_by_regexes(host_name, host_names) + if match == -1: + continue + profile_points += match + profile_value.append(bool(match)) + + # Task filtering + task_names = profile.get("tasks") + match = self.validate_value_by_regexes(task_name, task_names) + if match == -1: + continue + profile_points += match + profile_value.append(bool(match)) + + # Family filtering + families = profile.get("families") + match = self.validate_value_by_regexes(family, families) + if match == -1: + continue + profile_points += match + profile_value.append(bool(match)) + + if profile_points > highest_points: + matching_profiles = [] + highest_points = profile_points + + if profile_points == highest_points: + profile["__value__"] = profile_value + matching_profiles.append(profile) + + if not matching_profiles: + return + + if len(matching_profiles) == 1: + return matching_profiles[0] + + return self.profile_exclusion(profile) + + def profile_exclusion(self, matching_profiles): + """Find out most matching profile by host, task and family match. + + Profiles are selectivelly filtered. Each profile should have + "__value__" key with list of booleans. Each boolean represents + existence of filter for specific key (host, taks, family). + Profiles are looped in sequence. In each sequence are split into + true_list and false_list. For next sequence loop are used profiles in + true_list if there are any profiles else false_list is used. + + Filtering ends when only one profile left in true_list. Or when all + existence booleans loops passed, in that case first profile from left + profiles is returned. + + Args: + matching_profiles (list): Profiles with same values. + + Returns: + dict: Most matching profile. + """ + self.log.info( + "Search for first most matching profile in match order:" + " Host name -> Task name -> Family." + ) + # Filter all profiles with highest points value. First filter profiles + # with matching host if there are any then filter profiles by task + # name if there are any and lastly filter by family. Else use first in + # list. + idx = 0 + final_profile = None + while True: + profiles_true = [] + profiles_false = [] + for profile in matching_profiles: + value = profile["__value__"] + # Just use first profile when idx is greater than values. + if not idx < len(value): + final_profile = profile + break + + if value[idx]: + profiles_true.append(profile) + else: + profiles_false.append(profile) + + if final_profile is not None: + break + + if profiles_true: + matching_profiles = profiles_true + else: + matching_profiles = profiles_false + + if len(matching_profiles) == 1: + final_profile = matching_profiles[0] + break + idx += 1 + + final_profile.pop("__value__") + return final_profile + + def filter_burnins_by_families(self, profile, instance): + """Filter outputs that are not supported for instance families. + + Output definitions without families filter are marked as valid. + + Args: + profile (dict): Profile from presets matching current context. + families (list): All families of current instance. + + Returns: + list: Containg all output definitions matching entered families. + """ + filtered_burnin_defs = {} + + burnin_defs = profile.get("burnins") + if not burnin_defs: + return filtered_burnin_defs + + # Prepare families + families = self.families_from_instance(instance) + families = [family.lower() for family in families] + + for filename_suffix, burnin_def in burnin_defs.items(): + burnin_filter = burnin_def.get("filter") + # When filters not set then skip filtering process + if burnin_filter: + families_filters = burnin_filter.get("families") + if not self.families_filter_validation( + families, families_filters + ): + continue + + filtered_burnin_defs[filename_suffix] = burnin_def + return filtered_burnin_defs + + def families_filter_validation(self, families, output_families_filter): + """Determines if entered families intersect with families filters. + + All family values are lowered to avoid unexpected results. + """ + if not output_families_filter: + return True + + for family_filter in output_families_filter: + if not family_filter: + continue + + if not isinstance(family_filter, (list, tuple)): + if family_filter.lower() not in families: + continue + return True + + valid = True + for family in family_filter: + if family.lower() not in families: + valid = False + break + + if valid: + return True + return False + + def compile_list_of_regexes(self, in_list): + """Convert strings in entered list to compiled regex objects.""" + regexes = [] + if not in_list: + return regexes + + for item in in_list: + if not item: + continue + + if not isinstance(item, StringType): + self.log.warning(( + "Invalid type \"{}\" value \"{}\"." + " Expected . Skipping." + ).format(str(type(item)), str(item))) + continue + + regexes.append(re.compile(item)) + return regexes + + def validate_value_by_regexes(self, value, in_list): + """Validates in any regexe from list match entered value. + + Args: + in_list (list): List with regexes. + value (str): String where regexes is checked. + + Returns: + int: Returns `0` when list is not set or is empty. Returns `1` when + any regex match value and returns `-1` when none of regexes + match value entered. + """ + if not in_list: + return 0 + + output = -1 + regexes = self.compile_list_of_regexes(in_list) + for regex in regexes: + if re.match(regex, value): + output = 1 + break + return output + + def main_family_from_instance(self, instance): + """Returns main family of entered instance.""" + family = instance.data.get("family") + if not family: + family = instance.data["families"][0] + return family + + def families_from_instance(self, instance): + """Returns all families of entered instance.""" + families = [] + family = instance.data.get("family") + if family: + families.append(family) + + for family in (instance.data.get("families") or tuple()): + if family not in families: + families.append(family) + return families + + def burnin_script_path(self): + """Returns path to python script for burnin processing.""" + # TODO maybe convert to Plugin's attribute + # Get script path. + module_path = os.environ["PYPE_MODULE_ROOT"] + + # There can be multiple paths in PYPE_MODULE_ROOT, in which case + # we just take first one. + if os.pathsep in module_path: + module_path = module_path.split(os.pathsep)[0] + + scriptpath = os.path.normpath( + os.path.join( + module_path, + "pype", + "scripts", + "otio_burnin.py" + ) + ) + + self.log.debug("scriptpath: {}".format(scriptpath)) + + return scriptpath + + def python_executable_path(self): + """Returns path to Python 3 executable.""" + # TODO maybe convert to Plugin's attribute + # Get executable. + executable = os.getenv("PYPE_PYTHON_EXE") + + # There can be multiple paths in PYPE_PYTHON_EXE, in which case + # we just take first one. + if os.pathsep in executable: + executable = executable.split(os.pathsep)[0] + + self.log.debug("EXE: {}".format(executable)) + return executable + + def legacy_process(self, instance): context_data = instance.context.data version = instance.data.get( From fa97e76e329361dfc2ea41a9bc29c3ed961d2ccc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 13:53:41 +0200 Subject: [PATCH 24/71] added logs to legacy processing methods --- pype/plugins/global/publish/extract_burnin.py | 2 ++ pype/plugins/global/publish/extract_review.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 693940accfa..2f5d23a676f 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -761,6 +761,8 @@ def python_executable_path(self): return executable def legacy_process(self, instance): + self.log.warning("Legacy burnin presets are used.") + context_data = instance.context.data version = instance.data.get( diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index e7eb210b2df..c5f86e77065 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1074,6 +1074,8 @@ def filter_outputs_by_tags(self, outputs, tags): return filtered_outputs def legacy_process(self, instance): + self.log.warning("Legacy review presets are used.") + output_profiles = self.outputs or {} inst_data = instance.data From 4c7bacdb72cab7dfa3995b890a8518387cc1ec3d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 15:11:12 +0200 Subject: [PATCH 25/71] Fixed few bugs in extract review --- pype/plugins/global/publish/extract_review.py | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index c5f86e77065..063a3cb25e4 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -60,17 +60,21 @@ def process(self, instance): instance.data["representations"].remove(repre) def main_process(self, instance): - host_name = pyblish.api.registered_hosts()[-1].title() + host_name = pyblish.api.registered_hosts()[-1] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) + self.log.info("Host: \"{}\"".format(host_name)) + self.log.info("Task: \"{}\"".format(task_name)) + self.log.info("Family: \"{}\"".format(family)) + profile = self.find_matching_profile( host_name, task_name, family ) if not profile: self.log.info(( "Skipped instance. None of profiles in presets are for" - " Host: \"{host}\" | Family: \"{family}\" | Task \"{task}\"" + " Host: \"{}\" | Family: \"{}\" | Task \"{}\"" ).format(host_name, family, task_name)) return @@ -155,7 +159,7 @@ def main_process(self, instance): temp_data = self.prepare_temp_data(instance, repre, output_def) ffmpeg_args = self._ffmpeg_arguments( - output_def, instance, temp_data + output_def, instance, new_repre, temp_data ) subprcs_cmd = " ".join(ffmpeg_args) @@ -181,7 +185,9 @@ def main_process(self, instance): new_repre.pop("thumbnail", None) # adding representation - self.log.debug("Adding: {}".format(new_repre)) + self.log.debug( + "Adding new representation: {}".format(new_repre) + ) instance.data["representations"].append(new_repre) def input_is_sequence(self, repre): @@ -602,6 +608,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): # Skip processing if both conditions are not met if "reformat" not in new_repre["tags"] and not letter_box: + self.log.debug('Tag "reformat" and "letter_box" key are not set.') new_repre["resolutionWidth"] = input_width new_repre["resolutionHeight"] = input_height return filters @@ -616,9 +623,14 @@ def rescaling_filters(self, temp_data, output_def, new_repre): # Use source's input resolution instance does not have set it. if output_width is None or output_height is None: + self.log.debug("Using resolution from input.") output_width = input_width output_height = input_height + self.log.debug( + "Output resolution is {}x{}".format(output_width, output_height) + ) + # defining image ratios input_res_ratio = ( (float(input_width) * pixel_aspect) / input_height @@ -744,18 +756,24 @@ def lut_filters(self, new_repre, instance, input_args): def ffprobe_streams(self, path_to_file): """Load streams from entered filepath.""" + self.log.info( + "Getting information about input \"{}\".".format(path_to_file) + ) args = [ self.ffprobe_path, "-v quiet", "-print_format json", "-show_format", - "-show_streams" + "-show_streams", "\"{}\"".format(path_to_file) ] command = " ".join(args) + self.log.debug("FFprobe command: \"{}\"".format(command)) popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) - return json.loads(popen.communicate()[0])["streams"][0] + output = popen.communicate()[0] + self.log.debug("FFprobe output: {}".format(output)) + return json.loads(output)["streams"][0] def main_family_from_instance(self, instance): """Returns main family of entered instance.""" @@ -786,14 +804,14 @@ def compile_list_of_regexes(self, in_list): if not item: continue - if not isinstance(item, StringType): + try: + regexes.append(re.compile(item)) + except TypeError: self.log.warning(( "Invalid type \"{}\" value \"{}\"." - " Expected . Skipping." + " Expected string based object. Skipping." ).format(str(type(item)), str(item))) - continue - regexes.append(re.compile(item)) return regexes def validate_value_by_regexes(self, value, in_list): @@ -913,6 +931,9 @@ def find_matching_profile(self, host_name, task_name, family): host_names = profile.get("hosts") match = self.validate_value_by_regexes(host_name, host_names) if match == -1: + self.log.debug( + "\"{}\" not found in {}".format(host_name, host_names) + ) continue profile_points += match profile_value.append(bool(match)) @@ -921,6 +942,9 @@ def find_matching_profile(self, host_name, task_name, family): task_names = profile.get("tasks") match = self.validate_value_by_regexes(task_name, task_names) if match == -1: + self.log.debug( + "\"{}\" not found in {}".format(task_name, task_names) + ) continue profile_points += match profile_value.append(bool(match)) @@ -929,6 +953,9 @@ def find_matching_profile(self, host_name, task_name, family): families = profile.get("families") match = self.validate_value_by_regexes(family, families) if match == -1: + self.log.debug( + "\"{}\" not found in {}".format(family, families) + ) continue profile_points += match profile_value.append(bool(match)) @@ -936,13 +963,12 @@ def find_matching_profile(self, host_name, task_name, family): if profile_points < highest_profile_points: continue - profile["__value__"] = profile_value - if profile_points == highest_profile_points: - matching_profiles.append(profile) - - elif profile_points > highest_profile_points: - highest_profile_points = profile_points + if profile_points > highest_profile_points: matching_profiles = [] + highest_profile_points = profile_points + + if profile_points == highest_profile_points: + profile["__value__"] = profile_value matching_profiles.append(profile) if not matching_profiles: From 89d0bc7a4761ea158ad51fdfe394b51c5b3fee76 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 15:18:01 +0200 Subject: [PATCH 26/71] removed unused StringType --- pype/plugins/global/publish/extract_burnin.py | 10 ++++------ pype/plugins/global/publish/extract_review.py | 2 -- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 2f5d23a676f..ae53a16917c 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -6,8 +6,6 @@ import pype.api import pyblish -StringType = type("") - class ExtractBurnin(pype.api.Extractor): """ @@ -670,14 +668,14 @@ def compile_list_of_regexes(self, in_list): if not item: continue - if not isinstance(item, StringType): + try: + regexes.append(re.compile(item)) + except TypeError: self.log.warning(( "Invalid type \"{}\" value \"{}\"." - " Expected . Skipping." + " Expected string based object. Skipping." ).format(str(type(item)), str(item))) - continue - regexes.append(re.compile(item)) return regexes def validate_value_by_regexes(self, value, in_list): diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 063a3cb25e4..4eeb526a19b 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -8,8 +8,6 @@ import pype.api import pype.lib -StringType = type("") - class ExtractReview(pyblish.api.InstancePlugin): """Extracting Review mov file for Ftrack From 0c42f3d6a29efe7c1d6e6a8ef434839c08670f65 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 15:18:20 +0200 Subject: [PATCH 27/71] few fixes in ExtractBurnin --- pype/plugins/global/publish/extract_burnin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index ae53a16917c..c729c24908e 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -103,7 +103,7 @@ def main_process(self, instance): scriptpath = self.burnin_script_path() executable = self.python_executable_path() - for idx, repre in tuple(instance.data["representations"].items()): + for idx, repre in enumerate(tuple(instance.data["representations"])): self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) if not self.repres_is_valid(repre): continue @@ -324,9 +324,9 @@ def filter_burnins_by_tags(self, burnin_defs, tags): Returns: list: Containg all burnin definitions matching entered tags. """ - filtered_outputs = [] + filtered_burnins = {} repre_tags_low = [tag.lower() for tag in tags] - for burnin_def in burnin_defs: + for filename_suffix, burnin_def in burnin_defs.items(): valid = True output_filters = burnin_def.get("filter") if output_filters: @@ -344,9 +344,9 @@ def filter_burnins_by_tags(self, burnin_defs, tags): continue if valid: - filtered_outputs.append(burnin_def) + filtered_burnins[filename_suffix] = burnin_def - return filtered_outputs + return filtered_burnins def input_output_paths(self, new_repre, temp_data, filename_suffix): """Prepare input and output paths for representation. From 5ec1e510d4cce51a56819e8361384b4344ea102c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 15:59:21 +0200 Subject: [PATCH 28/71] fixed few bugs in extract burnin --- pype/plugins/global/publish/extract_burnin.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index c729c24908e..83b00bc5744 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -60,7 +60,8 @@ def process(self, instance): self.log.debug(instance.data["representations"]) def main_process(self, instance): - host_name = pyblish.api.registered_hosts()[-1].title() + # TODO get these data from context + host_name = pyblish.api.registered_hosts()[-1] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) @@ -160,11 +161,11 @@ def main_process(self, instance): if "delete" in new_repre["tags"]: new_repre["tags"].remove("delete") - # Update outputName to be able have multiple outputs + # Update name and outputName to be able have multiple outputs # Join previous "outputName" with filename suffix - new_repre["outputName"] = "_".join( - [new_repre["outputName"], filename_suffix] - ) + new_name = "_".join([new_repre["outputName"], filename_suffix]) + new_repre["name"] = new_name + new_repre["outputName"] = new_name # Prepare paths and files for process. self.input_output_paths(new_repre, temp_data, filename_suffix) @@ -288,7 +289,9 @@ def prepare_basic_data(self, instance): "frame_end_handle": frame_end_handle } - self.log.debug("Basic burnin_data: {}".format(burnin_data)) + self.log.debug( + "Basic burnin_data: {}".format(json.dumps(burnin_data, indent=4)) + ) return burnin_data, temp_data @@ -303,12 +306,16 @@ def repres_is_valid(self, repre): """ if "burnin" not in (repre.get("tags") or []): - self.log.info("Representation don't have \"burnin\" tag.") + self.log.info(( + "Representation \"{}\" don't have \"burnin\" tag. Skipped." + ).format(repre["name"])) return False # ffmpeg doesn't support multipart exrs if "multipartExr" in repre["tags"]: - self.log.info("Representation contain \"multipartExr\" tag.") + self.log.info(( + "Representation \"{}\" contain \"multipartExr\" tag. Skipped." + ).format(repre["name"])) return False return True @@ -417,7 +424,7 @@ def input_output_paths(self, new_repre, temp_data, filename_suffix): repre_files = output_filename temp_data["full_input_paths"] = full_input_paths - new_repre["repre_files"] = repre_files + new_repre["files"] = repre_files def prepare_repre_data(self, instance, repre, burnin_data, temp_data): """Prepare data for representation. @@ -755,7 +762,7 @@ def python_executable_path(self): if os.pathsep in executable: executable = executable.split(os.pathsep)[0] - self.log.debug("EXE: {}".format(executable)) + self.log.debug("executable: {}".format(executable)) return executable def legacy_process(self, instance): From 0fb7032e73f9cf3c45fd7cb6c0f30b6757166612 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 16:05:41 +0200 Subject: [PATCH 29/71] better script data log --- pype/plugins/global/publish/extract_burnin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 83b00bc5744..9db8023cbd7 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -179,7 +179,9 @@ def main_process(self, instance): "values": burnin_values } - self.log.debug("script_data: {}".format(script_data)) + self.log.debug( + "script_data: {}".format(json.dumps(script_data, indent=4)) + ) # Dump data to string dumped_script_data = json.dumps(script_data) From 498e9ab6e5dea13764b3ea2e2200b281c54d7988 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 16:10:38 +0200 Subject: [PATCH 30/71] fixed arguments in brunin script --- pype/scripts/otio_burnin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 4c9e0fc4d78..dc0e5fcbf6f 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -526,6 +526,6 @@ def burnins_from_data( in_data["output"], in_data["burnin_data"], codec_data=in_data.get("codec"), - options=in_data.get("optios"), - values=in_data.get("values") + options=in_data.get("options"), + burnin_values=in_data.get("values") ) From a5efeb616ccf087e44d8980919d5b9812150b835 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 16:36:36 +0200 Subject: [PATCH 31/71] fixed codec copy in burnin script --- pype/scripts/otio_burnin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index dc0e5fcbf6f..d1291184ae1 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -393,9 +393,6 @@ def burnins_from_data( "shot": "sh0010" } """ - # Make sure `codec_data` is list - if not codec_data: - codec_data = [] # Use legacy processing when options are not set if options is None or burnin_values is None: @@ -512,11 +509,14 @@ def burnins_from_data( text = value.format(**data) burnin.add_text(text, align, frame_start, frame_end) - codec_args = "" if codec_data: - codec_args = " ".join(codec_data) + # Use codec definition from method arguments + burnin_args = " ".join(codec_data) + else: + # Else use copy of source codecs for both audio and video + burnin_args = "-codec copy" - burnin.render(output_path, args=codec_args, overwrite=overwrite, **data) + burnin.render(output_path, args=burnin_args, overwrite=overwrite, **data) if __name__ == "__main__": From 69d0b8dd767b079da3de0e558e88d681779190e5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 18:47:20 +0200 Subject: [PATCH 32/71] added quotation marks where paths are used --- pype/scripts/otio_burnin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index d1291184ae1..a75df25255a 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -20,7 +20,7 @@ ).format(ffmpeg_path) FFPROBE = ( - '{} -v quiet -print_format json -show_format -show_streams %(source)s' + '{} -v quiet -print_format json -show_format -show_streams "%(source)s"' ).format(ffprobe_path) DRAWTEXT = ( @@ -55,7 +55,7 @@ def _streams(source): def get_fps(str_value): if str_value == "0/0": - print("Source has \"r_frame_rate\" value set to \"0/0\".") + log.warning("Source has \"r_frame_rate\" value set to \"0/0\".") return "Unknown" items = str_value.split("/") @@ -266,7 +266,7 @@ def command(self, output=None, args=None, overwrite=False): :returns: completed command :rtype: str """ - output = output or '' + output = '"{}"'.format(output or '') if overwrite: output = '-y {}'.format(output) From e1e3326dca2b2424f2e8990839abff2a819a7202 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 18:49:01 +0200 Subject: [PATCH 33/71] using codec from source since -codec copy can't be used --- pype/scripts/otio_burnin.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index a75df25255a..1a9f3e36052 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -509,14 +509,25 @@ def burnins_from_data( text = value.format(**data) burnin.add_text(text, align, frame_start, frame_end) + ffmpeg_args = [] if codec_data: # Use codec definition from method arguments - burnin_args = " ".join(codec_data) - else: - # Else use copy of source codecs for both audio and video - burnin_args = "-codec copy" + ffmpeg_args = codec_data - burnin.render(output_path, args=burnin_args, overwrite=overwrite, **data) + else: + codec_name = burnin._streams[0].get("codec_name") + log.info("codec_name: {}".format(codec_name)) + if codec_name: + ffmpeg_args.append("-codec:v {}".format(codec_name)) + + pix_fmt = burnin._streams[0].get("pix_fmt") + if pix_fmt: + ffmpeg_args.append("-pix_fmt {}".format(pix_fmt)) + + ffmpeg_args_str = " ".join(ffmpeg_args) + burnin.render( + output_path, args=ffmpeg_args_str, overwrite=overwrite, **data + ) if __name__ == "__main__": From 4ed64d9c8a1dbca0064cca0a70ff982d0815e5df Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 18:51:36 +0200 Subject: [PATCH 34/71] fixed logs in burnin script --- pype/scripts/otio_burnin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 1a9f3e36052..47e1811283a 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -300,7 +300,7 @@ def render(self, output, args=None, overwrite=False, **kwargs): args=args, overwrite=overwrite ) - # print(command) + # log.info(command) proc = subprocess.Popen(command, shell=True) proc.communicate() @@ -516,7 +516,6 @@ def burnins_from_data( else: codec_name = burnin._streams[0].get("codec_name") - log.info("codec_name: {}".format(codec_name)) if codec_name: ffmpeg_args.append("-codec:v {}".format(codec_name)) From 29b2196825fdc7a6ee5cd005842d3ee37263e8af Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 19:04:12 +0200 Subject: [PATCH 35/71] better handling of images and sequences --- pype/plugins/global/publish/extract_review.py | 66 +++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 4eeb526a19b..47ab1c482b4 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -284,14 +284,8 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): # Add argument to override output file ffmpeg_input_args.append("-y") - if temp_data["without_handles"]: - # NOTE used `-frames:v` instead of `-t` - should work the same way - duration_frames = ( - temp_data["output_frame_end"] - - temp_data["output_frame_start"] - + 1 - ) - ffmpeg_output_args.append("-frames:v {}".format(duration_frames)) + # Prepare input and output filepaths + self.input_output_paths(new_repre, output_def, temp_data) if temp_data["input_is_sequence"]: # Set start frame @@ -319,23 +313,37 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): start_sec = float(temp_data["handle_start"]) / temp_data["fps"] ffmpeg_input_args.append("-ss {:0.2f}".format(start_sec)) - full_input_path, full_output_path = self.input_output_paths( - new_repre, output_def, temp_data - ) - ffmpeg_input_args.append("-i \"{}\"".format(full_input_path)) + # Set output frames len to 1 when ouput is single image + if ( + temp_data["output_ext_is_image"] + and not temp_data["output_is_sequence"] + ): + output_frames_len = 1 - # Add audio arguments if there are any - audio_in_args, audio_filters, audio_out_args = self.audio_args( - instance, temp_data + else: + output_frames_len = ( + temp_data["output_frame_end"] + - temp_data["output_frame_start"] + + 1 + ) + + # NOTE used `-frames` instead of `-t` - should work the same way + # NOTE this also replaced `-shortest` argument + ffmpeg_output_args.append("-frames {}".format(output_frames_len)) + + # Add video/image input path + ffmpeg_input_args.append( + "-i \"{}\"".format(temp_data["full_input_path"]) ) - ffmpeg_input_args.extend(audio_in_args) - ffmpeg_audio_filters.extend(audio_filters) - ffmpeg_output_args.extend(audio_out_args) - # QUESTION what if audio is shoter than video? - # In case audio is longer than video`. - if "-shortest" not in ffmpeg_output_args: - ffmpeg_output_args.append("-shortest") + # Add audio arguments if there are any. Skipped when output are images. + if not temp_data["output_ext_is_image"]: + audio_in_args, audio_filters, audio_out_args = self.audio_args( + instance, temp_data + ) + ffmpeg_input_args.extend(audio_in_args) + ffmpeg_audio_filters.extend(audio_filters) + ffmpeg_output_args.extend(audio_out_args) res_filters = self.rescaling_filters(temp_data, output_def, new_repre) ffmpeg_video_filters.extend(res_filters) @@ -346,7 +354,9 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): ffmpeg_video_filters.extend(lut_filters) # NOTE This must be latest added item to output arguments. - ffmpeg_output_args.append("\"{}\"".format(full_output_path)) + ffmpeg_output_args.append( + "\"{}\"".format(temp_data["full_output_path"]) + ) return self.ffmpeg_full_args( ffmpeg_input_args, @@ -485,11 +495,11 @@ def input_output_paths(self, new_repre, output_def, temp_data): self.log.debug("New representation ext: `{}`".format(output_ext)) # Output is image file sequence witht frames + output_ext_is_image = output_ext in self.image_exts output_is_sequence = ( - (output_ext in self.image_exts) + output_ext_is_image and "sequence" in output_def["tags"] ) - if output_is_sequence: new_repre_files = [] frame_start = temp_data["output_frame_start"] @@ -534,11 +544,13 @@ def input_output_paths(self, new_repre, output_def, temp_data): temp_data["full_input_path_single_file"] = full_input_path_single_file temp_data["full_output_path"] = full_output_path + # Store information about output + temp_data["output_ext_is_image"] = output_ext_is_image + temp_data["output_is_sequence"] = output_is_sequence + self.log.debug("Input path {}".format(full_input_path)) self.log.debug("Output path {}".format(full_output_path)) - return full_input_path, full_output_path - def audio_args(self, instance, temp_data): """Prepares FFMpeg arguments for audio inputs.""" audio_in_args = [] From bd854fc7e177419036e5924039e43bd5ceed36db Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 19:04:43 +0200 Subject: [PATCH 36/71] extract review has better check for sequence input --- pype/plugins/global/publish/extract_burnin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 9db8023cbd7..b79cb15a696 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -375,7 +375,11 @@ def input_output_paths(self, new_repre, temp_data, filename_suffix): Returns: None: This is processing method. """ - is_sequence = "sequence" in new_repre["tags"] + # TODO we should find better way to know if input is sequence + is_sequence = ( + "sequence" in new_repre["tags"] + and isinstance(new_repre["files"], (tuple, list)) + ) if is_sequence: input_filename = new_repre["sequence_file"] else: From 7b9f29f5bf18fdbf1e7c20a09903b228ebb05cec Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 19:17:03 +0200 Subject: [PATCH 37/71] make sure output frames are integers --- pype/plugins/global/publish/extract_review.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 47ab1c482b4..7549b6818af 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -249,8 +249,8 @@ def prepare_temp_data(self, instance, repre, output_def): "handle_end": handle_end, "frame_start_handle": frame_start_handle, "frame_end_handle": frame_end_handle, - "output_frame_start": output_frame_start, - "output_frame_end": output_frame_end, + "output_frame_start": int(output_frame_start), + "output_frame_end": int(output_frame_end), "pixel_aspect": instance.data.get("pixelAspect", 1), "resolution_width": instance.data.get("resolutionWidth"), "resolution_height": instance.data.get("resolutionHeight"), From 54c3374977db7798d8d9813577e6a17e5b538e13 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 28 Apr 2020 19:17:16 +0200 Subject: [PATCH 38/71] fix join --- pype/plugins/global/publish/extract_burnin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index b79cb15a696..b2f858ee2f5 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -394,7 +394,7 @@ def input_output_paths(self, new_repre, temp_data, filename_suffix): frame_part = basename_parts.pop(-1) basename_start = ".".join(basename_parts) + filename_suffix - new_basename = ".".join(basename_start, frame_part) + new_basename = ".".join((basename_start, frame_part)) output_filename = new_basename + ext else: From 0ec66b337b8392efcfaf3edd30796549b9dbd839 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 29 Apr 2020 17:27:53 +0200 Subject: [PATCH 39/71] fix codec usage in extract slate review --- .../global/publish/extract_review_slate.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index e94701a312d..928e3fdc40a 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -107,7 +107,9 @@ def process(self, instance): output_args.extend(repre["outputDef"].get('output', [])) # Codecs are copied from source for whole input - output_args.append("-codec copy") + codec_args = self.codec_args(repre) + self.log.debug("Codec arguments: {}".format(codec_args)) + output_args.extend(codec_args) # make sure colors are correct output_args.extend([ @@ -269,3 +271,34 @@ def add_video_filter_args(self, args, inserting_arg): vf_back = "-vf " + ",".join(vf_fixed) return vf_back + + def codec_args(self, repre): + """Detect possible codec arguments from representation.""" + codec_args = [] + + # Get one filename of representation files + filename = repre["files"] + # If files is list then pick first filename in list + if isinstance(filename, (tuple, list)): + filename = filename[0] + # Get full path to the file + full_input_path = os.path.join(repre["stagingDir"], filename) + + try: + # Get information about input file via ffprobe tool + streams = pype.lib.ffprobe_streams(full_input_path) + except Exception: + self.log.warning( + "Could not get codec data from input.", + exc_info=True + ) + return codec_args + + codec_name = streams[0].get("codec_name") + if codec_name: + codec_args.append("-codec:v {}".format(codec_name)) + + pix_fmt = streams[0].get("pix_fmt") + if pix_fmt: + codec_args.append("-pix_fmt {}".format(pix_fmt)) + return codec_args From 19c436f8f2ef9552c3928c4f5ff5fbb5f097ea5e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 29 Apr 2020 17:28:50 +0200 Subject: [PATCH 40/71] ffprobe_streams moved to pype.lib --- pype/lib.py | 22 ++++++++++++++ pype/plugins/global/publish/extract_review.py | 29 ++----------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/pype/lib.py b/pype/lib.py index d3ccbc85890..7f88a130d33 100644 --- a/pype/lib.py +++ b/pype/lib.py @@ -1327,3 +1327,25 @@ def _collect_last_version_repres(self, asset_entities): ) return output + + +def ffprobe_streams(path_to_file): + """Load streams from entered filepath via ffprobe.""" + log.info( + "Getting information about input \"{}\".".format(path_to_file) + ) + args = [ + get_ffmpeg_tool_path("ffprobe"), + "-v quiet", + "-print_format json", + "-show_format", + "-show_streams", + "\"{}\"".format(path_to_file) + ] + command = " ".join(args) + log.debug("FFprobe command: \"{}\"".format(command)) + popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + + popen_output = popen.communicate()[0] + log.debug("FFprobe output: {}".format(popen_output)) + return json.loads(popen_output)["streams"] diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 7549b6818af..56a9c870b18 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -2,7 +2,6 @@ import re import copy import json -import subprocess import pyblish.api import clique import pype.api @@ -32,7 +31,6 @@ class ExtractReview(pyblish.api.InstancePlugin): # FFmpeg tools paths ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - ffprobe_path = pype.lib.get_ffmpeg_tool_path("ffprobe") # Preset attributes profiles = None @@ -83,7 +81,7 @@ def main_process(self, instance): profile, instance_families ) if not _profile_outputs: - self.log.info(( + self.log.warning(( "Skipped instance. All output definitions from selected" " profile does not match to instance families. \"{}\"" ).format(str(instance_families))) @@ -608,7 +606,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): # NOTE Skipped using instance's resolution full_input_path_single_file = temp_data["full_input_path_single_file"] - input_data = self.ffprobe_streams(full_input_path_single_file) + input_data = pype.lib.ffprobe_streams(full_input_path_single_file)[0] input_width = input_data["width"] input_height = input_data["height"] @@ -764,27 +762,6 @@ def lut_filters(self, new_repre, instance, input_args): return filters - def ffprobe_streams(self, path_to_file): - """Load streams from entered filepath.""" - self.log.info( - "Getting information about input \"{}\".".format(path_to_file) - ) - args = [ - self.ffprobe_path, - "-v quiet", - "-print_format json", - "-show_format", - "-show_streams", - "\"{}\"".format(path_to_file) - ] - command = " ".join(args) - self.log.debug("FFprobe command: \"{}\"".format(command)) - popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) - - output = popen.communicate()[0] - self.log.debug("FFprobe output: {}".format(output)) - return json.loads(output)["streams"][0] - def main_family_from_instance(self, instance): """Returns main family of entered instance.""" family = instance.data.get("family") @@ -982,7 +959,7 @@ def find_matching_profile(self, host_name, task_name, family): matching_profiles.append(profile) if not matching_profiles: - self.log.info(( + self.log.warning(( "None of profiles match your setup." " Host \"{}\" | Task: \"{}\" | Family: \"{}\"" ).format(host_name, task_name, family)) From c9dd40de22c94e80413b69aea9b6b7451f93ff29 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 29 Apr 2020 17:30:34 +0200 Subject: [PATCH 41/71] better repre debug log --- pype/plugins/global/publish/extract_review_slate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 928e3fdc40a..1825035aefa 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -30,7 +30,7 @@ def process(self, instance): fps = inst_data.get("fps") for idx, repre in enumerate(inst_data["representations"]): - self.log.debug("__ i: `{}`, repre: `{}`".format(idx, repre)) + self.log.debug("repre ({}): `{}`".format(idx + 1, repre)) p_tags = repre.get("tags", []) if "slate-frame" not in p_tags: From 623b4ac1e4a3c7f937b17d071c4c675ed0b3e719 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 29 Apr 2020 17:47:40 +0200 Subject: [PATCH 42/71] added logs to burnin script --- pype/scripts/otio_burnin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 47e1811283a..3bd022943c9 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -300,10 +300,10 @@ def render(self, output, args=None, overwrite=False, **kwargs): args=args, overwrite=overwrite ) - # log.info(command) + log.info("Launching command: {}".format(command)) proc = subprocess.Popen(command, shell=True) - proc.communicate() + log.info(proc.communicate()[0]) if proc.returncode != 0: raise RuntimeError("Failed to render '%s': %s'" % (output, command)) From 9291fda2c2c83d928b5f782657d92edbc48a529e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 29 Apr 2020 20:05:16 +0200 Subject: [PATCH 43/71] removed setting tags to families --- pype/plugins/global/publish/extract_review.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 56a9c870b18..ddd56124f15 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -146,12 +146,6 @@ def main_process(self, instance): "New representation tags: `{}`".format(new_repre["tags"]) ) - # # QUESTION Why the hell we were adding tags to families? - # # add families - # for tag in output_def["tags"]: - # if tag not in instance.data["families"]: - # instance.data["families"].append(tag) - temp_data = self.prepare_temp_data(instance, repre, output_def) ffmpeg_args = self._ffmpeg_arguments( From 9de5f00d695dced60b92a0a7069b0ab4c623160e Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 13 May 2020 22:42:09 +0200 Subject: [PATCH 44/71] wrong variable --- pype/plugins/global/publish/extract_burnin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index b61ed826452..f1c1bef2f5a 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -546,7 +546,7 @@ def find_matching_profile(self, host_name, task_name, family): if len(matching_profiles) == 1: return matching_profiles[0] - return self.profile_exclusion(profile) + return self.profile_exclusion(matching_profiles) def profile_exclusion(self, matching_profiles): """Find out most matching profile by host, task and family match. From b566741db928d7f5c6847f151b2431ed9bd21dcf Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 13 May 2020 22:42:27 +0200 Subject: [PATCH 45/71] change ftrack review tag from `preview` to `ftrackreview` --- pype/plugins/ftrack/publish/integrate_ftrack_instances.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py index 59fb507788e..11b569fd12e 100644 --- a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py +++ b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py @@ -63,7 +63,7 @@ def process(self, instance): "name": "thumbnail" # Default component name is "main". } comp['thumbnail'] = True - elif comp.get('preview') or ("preview" in comp.get('tags', [])): + elif comp.get('ftrackreview') or ("ftrackreview" in comp.get('tags', [])): ''' Ftrack bug requirement: - Start frame must be 0 From 91b44a2f9f8ca48cc4d1bd6eedb0e1ff5c647f23 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 14 May 2020 23:19:03 +0200 Subject: [PATCH 46/71] attempt to fix with doubled ffmpeg args --- pype/plugins/global/publish/extract_review.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 660c7290b78..c12a3a89df3 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -125,7 +125,8 @@ def main_process(self, instance): ).format(str(tags))) continue - for output_def in outputs: + for _output_def in outputs: + output_def = copy.deepcopy(_output_def) # Make sure output definition has "tags" key if "tags" not in output_def: output_def["tags"] = [] From 8ed2b664b47f67d1a8484be35a8650561bec66f8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 15 May 2020 10:03:56 +0200 Subject: [PATCH 47/71] only first output of extract burnin keep ftrackreview --- pype/plugins/global/publish/extract_burnin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index f1c1bef2f5a..2397912e56e 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -132,10 +132,18 @@ def main_process(self, instance): filled_anatomy = anatomy.format_all(burnin_data) burnin_data["anatomy"] = filled_anatomy.get_solved() + first_output = True + files_to_delete = [] for filename_suffix, burnin_def in repre_burnin_defs.items(): new_repre = copy.deepcopy(repre) + # Keep "ftrackreview" tag only on first output + if first_output: + first_output = False + elif "ftrackreview" in new_repre["tags"]: + new_repre["tags"].remove("ftrackreview") + burnin_options = copy.deepcopy(profile_options) burnin_values = copy.deepcopy(profile_burnins) From 1e0143a55c4b5074b4edff064b86407cd75c2ef3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 15 May 2020 10:41:02 +0200 Subject: [PATCH 48/71] modified few logs --- pype/plugins/global/publish/extract_review.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index c12a3a89df3..c235235c6aa 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -639,7 +639,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): (float(input_width) * pixel_aspect) / input_height ) output_res_ratio = float(output_width) / float(output_height) - self.log.debug("resolution_ratio: `{}`".format(input_res_ratio)) + self.log.debug("input_res_ratio: `{}`".format(input_res_ratio)) self.log.debug("output_res_ratio: `{}`".format(output_res_ratio)) # Round ratios to 2 decimal places for comparing @@ -700,13 +700,15 @@ def rescaling_filters(self, temp_data, output_def, new_repre): # scaling none square pixels and 1920 width if "reformat" in new_repre["tags"]: if input_res_ratio < output_res_ratio: - self.log.debug("lower then output") + self.log.debug( + "Input's resolution ratio is lower then output's" + ) width_scale = int(output_width * scale_factor_by_width) width_half_pad = int((output_width - width_scale) / 2) height_scale = output_height height_half_pad = 0 else: - self.log.debug("heigher then output") + self.log.debug("Input is heigher then output") width_scale = output_width width_half_pad = 0 height_scale = int(input_height * scale_factor_by_width) From 13bf4d9a118ce62f2772d0520aa566526263141f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 15 May 2020 14:53:46 +0200 Subject: [PATCH 49/71] skipped reformat tag --- pype/plugins/global/publish/extract_review.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index c235235c6aa..fc1e6377ef9 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -609,13 +609,6 @@ def rescaling_filters(self, temp_data, output_def, new_repre): self.log.debug("input_width: `{}`".format(input_width)) self.log.debug("resolution_height: `{}`".format(input_height)) - # Skip processing if both conditions are not met - if "reformat" not in new_repre["tags"] and not letter_box: - self.log.debug('Tag "reformat" and "letter_box" key are not set.') - new_repre["resolutionWidth"] = input_width - new_repre["resolutionHeight"] = input_height - return filters - # NOTE Setting only one of `width` or `heigth` is not allowed output_width = output_def.get("width") output_height = output_def.get("height") @@ -634,6 +627,21 @@ def rescaling_filters(self, temp_data, output_def, new_repre): "Output resolution is {}x{}".format(output_width, output_height) ) + # Skip processing if resolution is same as input's and letterbox is + # not set + if ( + output_width == input_width + and output_height == input_height + and not letter_box + ): + self.log.debug( + "Output resolution is same as input's" + " and \"letter_box\" key is not set. Skipping reformat part." + ) + new_repre["resolutionWidth"] = input_width + new_repre["resolutionHeight"] = input_height + return filters + # defining image ratios input_res_ratio = ( (float(input_width) * pixel_aspect) / input_height @@ -666,18 +674,12 @@ def rescaling_filters(self, temp_data, output_def, new_repre): if letter_box: ffmpeg_width = output_width ffmpeg_height = output_height - if "reformat" in new_repre["tags"]: - if input_res_ratio == output_res_ratio: - letter_box /= pixel_aspect - elif input_res_ratio < output_res_ratio: - letter_box /= scale_factor_by_width - else: - letter_box /= scale_factor_by_height - else: + if input_res_ratio == output_res_ratio: letter_box /= pixel_aspect - if input_res_ratio != output_res_ratio: - ffmpeg_width = input_width - ffmpeg_height = int(input_height * pixel_aspect) + elif input_res_ratio < output_res_ratio: + letter_box /= scale_factor_by_width + else: + letter_box /= scale_factor_by_height # QUESTION Is scale required when ffmpeg_width is same as # output_width and ffmpeg_height as output_height @@ -698,7 +700,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): filters.extend([scale_filter, "setsar=1", top_box, bottom_box]) # scaling none square pixels and 1920 width - if "reformat" in new_repre["tags"]: + if input_height != output_height or input_width != output_width: if input_res_ratio < output_res_ratio: self.log.debug( "Input's resolution ratio is lower then output's" From 46e3c540cde689e1ade23113cc16af3ba3bc0e13 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 18 May 2020 14:48:12 +0200 Subject: [PATCH 50/71] fixed multipartExr check --- pype/plugins/global/publish/extract_burnin.py | 19 ++++++++++++------- pype/plugins/global/publish/extract_review.py | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 2397912e56e..42b67891e9d 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -60,6 +60,18 @@ def process(self, instance): self.log.debug(instance.data["representations"]) def main_process(self, instance): + # ffmpeg doesn't support multipart exrs + if instance.data.get("multipartExr") is True: + instance_label = ( + getattr(instance, "label", None) + or instance.data.get("label") + or instance.data.get("name") + ) + self.log.info(( + "Instance \"{}\" contain \"multipartExr\". Skipped." + ).format(instance_label)) + return + # TODO get these data from context host_name = pyblish.api.registered_hosts()[-1] task_name = os.environ["AVALON_TASK"] @@ -320,13 +332,6 @@ def repres_is_valid(self, repre): "Representation \"{}\" don't have \"burnin\" tag. Skipped." ).format(repre["name"])) return False - - # ffmpeg doesn't support multipart exrs - if "multipartExr" in repre["tags"]: - self.log.info(( - "Representation \"{}\" contain \"multipartExr\" tag. Skipped." - ).format(repre["name"])) - return False return True def filter_burnins_by_tags(self, burnin_defs, tags): diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index fc1e6377ef9..d18cb9aef6f 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -56,6 +56,17 @@ def process(self, instance): instance.data["representations"].remove(repre) def main_process(self, instance): + if instance.data.get("multipartExr") is True: + instance_label = ( + getattr(instance, "label", None) + or instance.data.get("label") + or instance.data.get("name") + ) + self.log.info(( + "Instance \"{}\" contain \"multipartExr\". Skipped." + ).format(instance_label)) + return + host_name = pyblish.api.registered_hosts()[-1] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) @@ -96,11 +107,7 @@ def main_process(self, instance): # Loop through representations for repre in tuple(instance.data["representations"]): tags = repre.get("tags") or [] - if ( - "review" not in tags - or "multipartExr" in tags - or "thumbnail" in tags - ): + if "review" not in tags or "thumbnail" in tags: continue input_ext = repre["ext"] @@ -1122,7 +1129,7 @@ def legacy_process(self, instance): tags = repre.get("tags", []) - if "multipartExr" in tags: + if instance.data.get("multipartExr") is True: # ffmpeg doesn't support multipart exrs continue From 1d27c19897dd6a917e9250261eb3333c13ff449f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 18 May 2020 14:51:54 +0200 Subject: [PATCH 51/71] moved multipartExr much earlier --- pype/plugins/global/publish/extract_burnin.py | 24 +++++++++---------- pype/plugins/global/publish/extract_review.py | 23 +++++++++--------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 42b67891e9d..3930f4de9b2 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -42,6 +42,18 @@ class ExtractBurnin(pype.api.Extractor): fields = None def process(self, instance): + # ffmpeg doesn't support multipart exrs + if instance.data.get("multipartExr") is True: + instance_label = ( + getattr(instance, "label", None) + or instance.data.get("label") + or instance.data.get("name") + ) + self.log.info(( + "Instance \"{}\" contain \"multipartExr\". Skipped." + ).format(instance_label)) + return + # QUESTION what is this for and should we raise an exception? if "representations" not in instance.data: raise RuntimeError("Burnin needs already created mov to work on.") @@ -60,18 +72,6 @@ def process(self, instance): self.log.debug(instance.data["representations"]) def main_process(self, instance): - # ffmpeg doesn't support multipart exrs - if instance.data.get("multipartExr") is True: - instance_label = ( - getattr(instance, "label", None) - or instance.data.get("label") - or instance.data.get("name") - ) - self.log.info(( - "Instance \"{}\" contain \"multipartExr\". Skipped." - ).format(instance_label)) - return - # TODO get these data from context host_name = pyblish.api.registered_hosts()[-1] task_name = os.environ["AVALON_TASK"] diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index d18cb9aef6f..dee6729ecaf 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -42,6 +42,18 @@ class ExtractReview(pyblish.api.InstancePlugin): to_height = 1080 def process(self, instance): + # ffmpeg doesn't support multipart exrs + if instance.data.get("multipartExr") is True: + instance_label = ( + getattr(instance, "label", None) + or instance.data.get("label") + or instance.data.get("name") + ) + self.log.info(( + "Instance \"{}\" contain \"multipartExr\". Skipped." + ).format(instance_label)) + return + # Use legacy processing when `profiles` is not set. if self.profiles is None: return self.legacy_process(instance) @@ -56,17 +68,6 @@ def process(self, instance): instance.data["representations"].remove(repre) def main_process(self, instance): - if instance.data.get("multipartExr") is True: - instance_label = ( - getattr(instance, "label", None) - or instance.data.get("label") - or instance.data.get("name") - ) - self.log.info(( - "Instance \"{}\" contain \"multipartExr\". Skipped." - ).format(instance_label)) - return - host_name = pyblish.api.registered_hosts()[-1] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) From 4027dbfe833c8a85af54d6a58be89aa842eb3df4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 19 May 2020 20:57:11 +0200 Subject: [PATCH 52/71] override argument added before output file --- pype/plugins/global/publish/extract_review.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index dee6729ecaf..228adb46862 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -282,9 +282,6 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): ffmpeg_video_filters = out_def_ffmpeg_args.get("video_filters") or [] ffmpeg_audio_filters = out_def_ffmpeg_args.get("audio_filters") or [] - # Add argument to override output file - ffmpeg_input_args.append("-y") - # Prepare input and output filepaths self.input_output_paths(new_repre, output_def, temp_data) @@ -354,6 +351,9 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): lut_filters = self.lut_filters(new_repre, instance, ffmpeg_input_args) ffmpeg_video_filters.extend(lut_filters) + # Add argument to override output file + ffmpeg_output_args.append("-y") + # NOTE This must be latest added item to output arguments. ffmpeg_output_args.append( "\"{}\"".format(temp_data["full_output_path"]) From c0b85b71bd301689b05bb442d914a804f1561b0c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 19 May 2020 21:25:24 +0200 Subject: [PATCH 53/71] comments changes --- pype/plugins/global/publish/extract_review.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 228adb46862..8bf6ba36f54 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -93,7 +93,7 @@ def main_process(self, instance): profile, instance_families ) if not _profile_outputs: - self.log.warning(( + self.log.info(( "Skipped instance. All output definitions from selected" " profile does not match to instance families. \"{}\"" ).format(str(instance_families))) @@ -508,7 +508,7 @@ def input_output_paths(self, new_repre, output_def, temp_data): filename_base = "{}_{}".format(filename, filename_suffix) # Temporary tempalte for frame filling. Example output: - # "basename.%04d.mov" when `frame_end` == 1001 + # "basename.%04d.exr" when `frame_end` == 1001 repr_file = "{}.%{:0>2}d.{}".format( filename_base, len(str(frame_end)), output_ext ) @@ -678,10 +678,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): ) # letter_box - letter_box = output_def.get("letter_box") if letter_box: - ffmpeg_width = output_width - ffmpeg_height = output_height if input_res_ratio == output_res_ratio: letter_box /= pixel_aspect elif input_res_ratio < output_res_ratio: @@ -689,10 +686,8 @@ def rescaling_filters(self, temp_data, output_def, new_repre): else: letter_box /= scale_factor_by_height - # QUESTION Is scale required when ffmpeg_width is same as - # output_width and ffmpeg_height as output_height scale_filter = "scale={}x{}:flags=lanczos".format( - ffmpeg_width, ffmpeg_height + output_width, output_height ) top_box = ( From 152cfbb48e1286b318ff02cc88cf8cc9215b78b1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 20 May 2020 15:28:42 +0200 Subject: [PATCH 54/71] few minor fixes --- pype/plugins/global/publish/extract_review.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 8bf6ba36f54..e1a739b87c3 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -303,7 +303,7 @@ def _ffmpeg_arguments(self, output_def, instance, new_repre, temp_data): ) elif temp_data["without_handles"]: - # QUESTION Shall we change this to use filter: + # TODO use frames ubstead if `-ss`: # `select="gte(n\,{handle_start}),setpts=PTS-STARTPTS` # Pros: # 1.) Python is not good at float operation @@ -667,7 +667,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): float(output_width) / (input_width * pixel_aspect) ) scale_factor_by_height = ( - float(output_height) / (input_height * pixel_aspect) + float(output_height) / input_height ) self.log.debug( @@ -716,7 +716,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): self.log.debug("Input is heigher then output") width_scale = output_width width_half_pad = 0 - height_scale = int(input_height * scale_factor_by_width) + height_scale = int(input_height * scale_factor_by_height) height_half_pad = int((output_height - height_scale) / 2) self.log.debug("width_scale: `{}`".format(width_scale)) From 351d7cb03cd82c1be360425b666e8a20c63996b1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 20 May 2020 15:49:28 +0200 Subject: [PATCH 55/71] hopefully fixed issues with rescaling --- pype/plugins/global/publish/extract_review.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index e1a739b87c3..7321fa04ce4 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -615,7 +615,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): self.log.debug("pixel_aspect: `{}`".format(pixel_aspect)) self.log.debug("input_width: `{}`".format(input_width)) - self.log.debug("resolution_height: `{}`".format(input_height)) + self.log.debug("input_height: `{}`".format(input_height)) # NOTE Setting only one of `width` or `heigth` is not allowed output_width = output_def.get("width") @@ -703,12 +703,16 @@ def rescaling_filters(self, temp_data, output_def, new_repre): filters.extend([scale_filter, "setsar=1", top_box, bottom_box]) # scaling none square pixels and 1920 width - if input_height != output_height or input_width != output_width: + if ( + input_height != output_height + or input_width != output_width + or pixel_aspect != 1 + ): if input_res_ratio < output_res_ratio: self.log.debug( "Input's resolution ratio is lower then output's" ) - width_scale = int(output_width * scale_factor_by_width) + width_scale = int(output_width * scale_factor_by_height) width_half_pad = int((output_width - width_scale) / 2) height_scale = output_height height_half_pad = 0 @@ -716,7 +720,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): self.log.debug("Input is heigher then output") width_scale = output_width width_half_pad = 0 - height_scale = int(input_height * scale_factor_by_height) + height_scale = int(input_height * scale_factor_by_width) height_half_pad = int((output_height - height_scale) / 2) self.log.debug("width_scale: `{}`".format(width_scale)) From 3d7da50191c8dc990a47d949aca9e807df3f3a10 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 20 May 2020 16:00:46 +0200 Subject: [PATCH 56/71] fix scale in legacy code --- pype/plugins/global/publish/extract_review.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 7321fa04ce4..4958a972263 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1317,6 +1317,10 @@ def legacy_process(self, instance): if resolution_ratio_test < delivery_ratio_test: scale_factor = float(self.to_width) / ( resolution_width * pixel_aspect) + if int(scale_factor * 100) == 100: + scale_factor = ( + float(self.to_height) / resolution_height + ) self.log.debug("__ scale_factor: `{}`".format(scale_factor)) From f4a01576480a25ac807d7ed5b7eb5f9ca183d7ab Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 20 May 2020 16:03:55 +0200 Subject: [PATCH 57/71] updated legacy code with latest develop --- pype/plugins/global/publish/extract_review.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 4958a972263..119f8804f79 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1148,6 +1148,12 @@ def legacy_process(self, instance): repre_new = repre.copy() ext = profile.get("ext", None) p_tags = profile.get('tags', []) + + # append repre tags into profile tags + for t in tags: + if t not in p_tags: + p_tags.append(t) + self.log.info("p_tags: `{}`".format(p_tags)) # adding control for presets to be sequence From 774d5deacfa79ff7375ca2c102812a39cb1ce785 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 20 May 2020 16:08:45 +0200 Subject: [PATCH 58/71] also check pixel_aspect in skipping part --- pype/plugins/global/publish/extract_review.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index abbc4fc5956..5f8330d900d 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -641,6 +641,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): output_width == input_width and output_height == input_height and not letter_box + and pixel_aspect == 1 ): self.log.debug( "Output resolution is same as input's" From 0a07803007520256c5ecf88b7acfa129bd943369 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 20 May 2020 16:28:03 +0200 Subject: [PATCH 59/71] removed doubled process method --- pype/plugins/global/publish/extract_review.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 5f8330d900d..cdebb38f987 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1096,7 +1096,6 @@ def filter_outputs_by_tags(self, outputs, tags): def legacy_process(self, instance): self.log.warning("Legacy review presets are used.") - def process(self, instance): output_profiles = self.outputs or {} inst_data = instance.data From 3a6ab5496324cf2741b682ae3a8904304699cf01 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 10:05:11 +0200 Subject: [PATCH 60/71] changed comment and docstring by comments --- pype/plugins/global/publish/extract_review.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index cdebb38f987..e40e9051181 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1022,7 +1022,7 @@ def families_filter_validation(self, families, output_families_filter): return False def filter_outputs_by_families(self, profile, families): - """Filter outputs that are not supported for instance families. + """Return outputs matching input instance families. Output definitions without families filter are marked as valid. @@ -1044,7 +1044,8 @@ def filter_outputs_by_families(self, profile, families): filtered_outputs = {} for filename_suffix, output_def in outputs.items(): output_filters = output_def.get("filter") - # When filters not set then skip filtering process + # If no filter on output preset, skip filtering and add output + # profile for farther processing if not output_filters: filtered_outputs[filename_suffix] = output_def continue From fe777569db5c9177d5269a193e4cd091239623c4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 10:05:44 +0200 Subject: [PATCH 61/71] boolean variables are easier to see now by comments --- pype/plugins/global/publish/extract_review.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index e40e9051181..c0fce645b51 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -234,7 +234,7 @@ def prepare_temp_data(self, instance, repre, output_def): frame_end_handle = frame_end + handle_end # Change output frames when output should be without handles - without_handles = "no-handles" in output_def["tags"] + without_handles = bool("no-handles" in output_def["tags"]) if without_handles: output_frame_start = frame_start output_frame_end = frame_end @@ -496,8 +496,8 @@ def input_output_paths(self, new_repre, output_def, temp_data): self.log.debug("New representation ext: `{}`".format(output_ext)) # Output is image file sequence witht frames - output_ext_is_image = output_ext in self.image_exts - output_is_sequence = ( + output_ext_is_image = bool(output_ext in self.image_exts) + output_is_sequence = bool( output_ext_is_image and "sequence" in output_def["tags"] ) From 179f0e2e464c249fdf88c48e9c79feb1d7d238c8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 15:41:59 +0200 Subject: [PATCH 62/71] update changes from PR `extract review reformat issue #166` --- pype/nuke/lib.py | 1 + pype/plugins/global/publish/extract_review.py | 2 +- pype/plugins/global/publish/extract_review_slate.py | 12 +++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index a706753755e..ade7e966911 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -1669,6 +1669,7 @@ def generate_mov(self, farm=False): if any(colorspaces): # OCIOColorSpace with controled output dag_node = nuke.createNode("OCIOColorSpace") + self._temp_nodes.append(dag_node) for c in colorspaces: test = dag_node["out_colorspace"].setValue(str(c)) if test: diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index c0fce645b51..09899657588 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1323,7 +1323,7 @@ def legacy_process(self, instance): delivery_ratio_test = float( "{:0.2f}".format(delivery_ratio)) - if resolution_ratio_test < delivery_ratio_test: + if resolution_ratio_test != delivery_ratio_test: scale_factor = float(self.to_width) / ( resolution_width * pixel_aspect) if int(scale_factor * 100) == 100: diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 1825035aefa..3db4b2e97e2 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -67,9 +67,15 @@ def process(self, instance): delivery_ratio_test = float( "{:0.2f}".format(delivery_ratio)) - if resolution_ratio_test < delivery_ratio_test: - scale_factor = float(to_width) / ( - resolution_width * pixel_aspect) + if resolution_ratio_test != delivery_ratio_test: + scale_factor = ( + float(to_width) / ( + resolution_width * pixel_aspect) + ) + if int(scale_factor * 100) == 100: + scale_factor = ( + float(to_height) / resolution_height + ) self.log.debug("__ scale_factor: `{}`".format(scale_factor)) From 5203bb31366343cdcf42da5b8be89085075a61b2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 17:05:17 +0200 Subject: [PATCH 63/71] add profile from input if is set in extract burnin --- pype/scripts/otio_burnin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index c35ce27b9cb..138165d4894 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -519,6 +519,12 @@ def burnins_from_data( if codec_name: ffmpeg_args.append("-codec:v {}".format(codec_name)) + profile_name = burnin._streams[0].get("profile") + if profile_name: + # lower profile name and repalce spaces with underscore + profile_name = profile_name.replace(" ", "_").lower() + ffmpeg_args.append("-profile:v {}".format(profile_name)) + pix_fmt = burnin._streams[0].get("pix_fmt") if pix_fmt: ffmpeg_args.append("-pix_fmt {}".format(pix_fmt)) From b5fbf1a81f9ade3ef33e2f5b9b5e5c3d2fe735ee Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 18:27:38 +0200 Subject: [PATCH 64/71] use slate's inputs resolution instead of using resolution from instance or representation --- pype/plugins/global/publish/extract_review_slate.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 3db4b2e97e2..46e9dfc6f89 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -26,6 +26,10 @@ def process(self, instance): slate_path = inst_data.get("slateFrame") ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + slate_stream = pype.lib.ffprobe_streams(slate_path)[0] + resolution_width = slate_stream["width"] + resolution_height = slate_stream["height"] + pixel_aspect = inst_data.get("pixelAspect", 1) fps = inst_data.get("fps") @@ -40,15 +44,6 @@ def process(self, instance): to_width = repre["resolutionWidth"] to_height = repre["resolutionHeight"] - # QUESTION Should we use resolution from instance and not source's? - resolution_width = inst_data.get("resolutionWidth") - if resolution_width is None: - resolution_width = to_width - - resolution_height = inst_data.get("resolutionHeight") - if resolution_height is None: - resolution_height = to_height - # defining image ratios resolution_ratio = ( (float(resolution_width) * pixel_aspect) / resolution_height From 8986839053d0d4ab22e93d7a8dc3f67a23279269 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 18:28:36 +0200 Subject: [PATCH 65/71] do not use output arguments from output definition --- pype/plugins/global/publish/extract_review_slate.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 46e9dfc6f89..3f717cfd96d 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -103,9 +103,6 @@ def process(self, instance): "-t 0.04"] ) - # output args - # preset's output data - output_args.extend(repre["outputDef"].get('output', [])) # Codecs are copied from source for whole input codec_args = self.codec_args(repre) @@ -299,6 +296,11 @@ def codec_args(self, repre): if codec_name: codec_args.append("-codec:v {}".format(codec_name)) + profile_name = streams[0].get("profile") + if profile_name: + profile_name = profile_name.replace(" ", "_").lower() + codec_args.append("-profile:v {}".format(profile_name)) + pix_fmt = streams[0].get("pix_fmt") if pix_fmt: codec_args.append("-pix_fmt {}".format(pix_fmt)) From 8645ee13ce7b0cc746c47cb38ede3bf0db22dfa4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 18:30:33 +0200 Subject: [PATCH 66/71] moved -y ffmpeg arg to output args --- pype/plugins/global/publish/extract_review_slate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 3f717cfd96d..95f420b1ed6 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -93,8 +93,7 @@ def process(self, instance): input_args = [] output_args = [] - # overrides output file - input_args.append("-y") + # preset's input data input_args.extend(repre["outputDef"].get('input', [])) input_args.append("-loop 1 -i {}".format(slate_path)) @@ -103,7 +102,6 @@ def process(self, instance): "-t 0.04"] ) - # Codecs are copied from source for whole input codec_args = self.codec_args(repre) self.log.debug("Codec arguments: {}".format(codec_args)) @@ -157,6 +155,8 @@ def process(self, instance): output_args, scaling_arg) # add it to output_args output_args.insert(0, vf_back) + # overrides output file + output_args.append("-y") slate_v_path = slate_path.replace(".png", ext) output_args.append(slate_v_path) From f5e66c87bbd475f0597af823d8f70b3f4fb2f00a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 19:07:23 +0200 Subject: [PATCH 67/71] slate is scaled all the time to same resolution as review has --- .../global/publish/extract_review_slate.py | 104 ++++++++---------- 1 file changed, 44 insertions(+), 60 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 95f420b1ed6..8426ae84eb6 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -27,8 +27,8 @@ def process(self, instance): ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") slate_stream = pype.lib.ffprobe_streams(slate_path)[0] - resolution_width = slate_stream["width"] - resolution_height = slate_stream["height"] + slate_width = slate_stream["width"] + slate_height = slate_stream["height"] pixel_aspect = inst_data.get("pixelAspect", 1) fps = inst_data.get("fps") @@ -46,33 +46,28 @@ def process(self, instance): # defining image ratios resolution_ratio = ( - (float(resolution_width) * pixel_aspect) / resolution_height + (float(slate_width) * pixel_aspect) / slate_height ) delivery_ratio = float(to_width) / float(to_height) self.log.debug("resolution_ratio: `{}`".format(resolution_ratio)) self.log.debug("delivery_ratio: `{}`".format(delivery_ratio)) # get scale factor - scale_factor = float(to_height) / ( - resolution_height * pixel_aspect) + scale_factor_by_height = float(to_height) / slate_height + scale_factor_by_width = float(to_width) / ( + slate_width * pixel_aspect + ) # shorten two decimals long float number for testing conditions - resolution_ratio_test = float( - "{:0.2f}".format(resolution_ratio)) - delivery_ratio_test = float( - "{:0.2f}".format(delivery_ratio)) - - if resolution_ratio_test != delivery_ratio_test: - scale_factor = ( - float(to_width) / ( - resolution_width * pixel_aspect) - ) - if int(scale_factor * 100) == 100: - scale_factor = ( - float(to_height) / resolution_height - ) - - self.log.debug("__ scale_factor: `{}`".format(scale_factor)) + resolution_ratio_test = float("{:0.2f}".format(resolution_ratio)) + delivery_ratio_test = float("{:0.2f}".format(delivery_ratio)) + + self.log.debug("__ scale_factor_by_width: `{}`".format( + scale_factor_by_width + )) + self.log.debug("__ scale_factor_by_height: `{}`".format( + scale_factor_by_height + )) _remove_at_end = [] @@ -116,45 +111,34 @@ def process(self, instance): ]) # scaling none square pixels and 1920 width - if "reformat" in p_tags: - if resolution_ratio_test < delivery_ratio_test: - self.log.debug("lower then delivery") - width_scale = int(to_width * scale_factor) - width_half_pad = int(( - to_width - width_scale) / 2) - height_scale = to_height - height_half_pad = 0 - else: - self.log.debug("heigher then delivery") - width_scale = to_width - width_half_pad = 0 - scale_factor = float(to_width) / (float( - resolution_width) * pixel_aspect) - self.log.debug(scale_factor) - height_scale = int( - resolution_height * scale_factor) - height_half_pad = int( - (to_height - height_scale) / 2) - - self.log.debug( - "__ width_scale: `{}`".format(width_scale)) - self.log.debug( - "__ width_half_pad: `{}`".format(width_half_pad)) - self.log.debug( - "__ height_scale: `{}`".format(height_scale)) - self.log.debug( - "__ height_half_pad: `{}`".format(height_half_pad)) - - scaling_arg = ("scale={0}x{1}:flags=lanczos," - "pad={2}:{3}:{4}:{5}:black,setsar=1").format( - width_scale, height_scale, to_width, to_height, - width_half_pad, height_half_pad - ) - - vf_back = self.add_video_filter_args( - output_args, scaling_arg) - # add it to output_args - output_args.insert(0, vf_back) + if resolution_ratio_test < delivery_ratio_test: + self.log.debug("lower then delivery") + width_scale = int(slate_width * scale_factor_by_height) + width_half_pad = int((to_width - width_scale) / 2) + height_scale = to_height + height_half_pad = 0 + else: + self.log.debug("heigher then delivery") + width_scale = to_width + width_half_pad = 0 + height_scale = int(slate_height * scale_factor_by_width) + height_half_pad = int((to_height - height_scale) / 2) + + self.log.debug("__ width_scale: `{}`".format(width_scale)) + self.log.debug("__ width_half_pad: `{}`".format(width_half_pad)) + self.log.debug("__ height_scale: `{}`".format(height_scale)) + self.log.debug("__ height_half_pad: `{}`".format(height_half_pad)) + + scaling_arg = ("scale={0}x{1}:flags=lanczos," + "pad={2}:{3}:{4}:{5}:black,setsar=1").format( + width_scale, height_scale, to_width, to_height, + width_half_pad, height_half_pad + ) + + vf_back = self.add_video_filter_args(output_args, scaling_arg) + # add it to output_args + output_args.insert(0, vf_back) + # overrides output file output_args.append("-y") From e5477ddc75e2d00780461c492bd8c86666e7035b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 19:07:57 +0200 Subject: [PATCH 68/71] fixed variable in extract review rescaling --- pype/plugins/global/publish/extract_review.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 09899657588..5b35e727ac6 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -713,7 +713,7 @@ def rescaling_filters(self, temp_data, output_def, new_repre): self.log.debug( "Input's resolution ratio is lower then output's" ) - width_scale = int(output_width * scale_factor_by_height) + width_scale = int(input_width * scale_factor_by_height) width_half_pad = int((output_width - width_scale) / 2) height_scale = output_height height_half_pad = 0 From b92509067038c9dd7bcd796bc1dc650f54390aef Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 21 May 2020 19:20:34 +0200 Subject: [PATCH 69/71] extract review slate should be backwards compatible --- .../global/publish/extract_review_slate.py | 91 +++++++++++++------ 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 8426ae84eb6..f2ea6c08750 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -30,6 +30,11 @@ def process(self, instance): slate_width = slate_stream["width"] slate_height = slate_stream["height"] + if "reviewToWidth" in inst_data: + use_legacy_code = True + else: + use_legacy_code = False + pixel_aspect = inst_data.get("pixelAspect", 1) fps = inst_data.get("fps") @@ -41,8 +46,12 @@ def process(self, instance): continue # values are set in ExtractReview - to_width = repre["resolutionWidth"] - to_height = repre["resolutionHeight"] + if use_legacy_code: + to_width = inst_data["reviewToWidth"] + to_height = inst_data["reviewToHeight"] + else: + to_width = repre["resolutionWidth"] + to_height = repre["resolutionHeight"] # defining image ratios resolution_ratio = ( @@ -90,17 +99,25 @@ def process(self, instance): output_args = [] # preset's input data - input_args.extend(repre["outputDef"].get('input', [])) + if use_legacy_code: + input_args.extend(repre["_profile"].get('input', [])) + else: + input_args.extend(repre["outputDef"].get('input', [])) input_args.append("-loop 1 -i {}".format(slate_path)) input_args.extend([ "-r {}".format(fps), "-t 0.04"] ) - # Codecs are copied from source for whole input - codec_args = self.codec_args(repre) - self.log.debug("Codec arguments: {}".format(codec_args)) - output_args.extend(codec_args) + if use_legacy_code: + codec_args = repre["_profile"].get('codec', []) + output_args.extend(codec_args) + # preset's output data + output_args.extend(repre["_profile"].get('output', [])) + else: + # Codecs are copied from source for whole input + codec_args = self.codec_args(repre) + output_args.extend(codec_args) # make sure colors are correct output_args.extend([ @@ -111,29 +128,43 @@ def process(self, instance): ]) # scaling none square pixels and 1920 width - if resolution_ratio_test < delivery_ratio_test: - self.log.debug("lower then delivery") - width_scale = int(slate_width * scale_factor_by_height) - width_half_pad = int((to_width - width_scale) / 2) - height_scale = to_height - height_half_pad = 0 - else: - self.log.debug("heigher then delivery") - width_scale = to_width - width_half_pad = 0 - height_scale = int(slate_height * scale_factor_by_width) - height_half_pad = int((to_height - height_scale) / 2) - - self.log.debug("__ width_scale: `{}`".format(width_scale)) - self.log.debug("__ width_half_pad: `{}`".format(width_half_pad)) - self.log.debug("__ height_scale: `{}`".format(height_scale)) - self.log.debug("__ height_half_pad: `{}`".format(height_half_pad)) - - scaling_arg = ("scale={0}x{1}:flags=lanczos," - "pad={2}:{3}:{4}:{5}:black,setsar=1").format( - width_scale, height_scale, to_width, to_height, - width_half_pad, height_half_pad - ) + if ( + # Always scale slate if not legacy + not use_legacy_code or + # Legacy code required reformat tag + (use_legacy_code and "reformat" in p_tags) + ): + if resolution_ratio_test < delivery_ratio_test: + self.log.debug("lower then delivery") + width_scale = int(slate_width * scale_factor_by_height) + width_half_pad = int((to_width - width_scale) / 2) + height_scale = to_height + height_half_pad = 0 + else: + self.log.debug("heigher then delivery") + width_scale = to_width + width_half_pad = 0 + height_scale = int(slate_height * scale_factor_by_width) + height_half_pad = int((to_height - height_scale) / 2) + + self.log.debug( + "__ width_scale: `{}`".format(width_scale) + ) + self.log.debug( + "__ width_half_pad: `{}`".format(width_half_pad) + ) + self.log.debug( + "__ height_scale: `{}`".format(height_scale) + ) + self.log.debug( + "__ height_half_pad: `{}`".format(height_half_pad) + ) + + scaling_arg = ("scale={0}x{1}:flags=lanczos," + "pad={2}:{3}:{4}:{5}:black,setsar=1").format( + width_scale, height_scale, to_width, to_height, + width_half_pad, height_half_pad + ) vf_back = self.add_video_filter_args(output_args, scaling_arg) # add it to output_args From 6e27c92800ccefb5b2be143234227b51f13162cd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 22 May 2020 14:07:12 +0200 Subject: [PATCH 70/71] fixed index access of iterable object --- pype/plugins/global/publish/integrate_new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 0cd46d8891f..bd908901cc3 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -743,13 +743,13 @@ def template_name_from_instance(self, instance): matching_profiles[name] = filters if len(matching_profiles) == 1: - template_name = matching_profiles.keys()[0] + template_name = tuple(matching_profiles.keys())[0] self.log.debug( "Using template name \"{}\".".format(template_name) ) elif len(matching_profiles) > 1: - template_name = matching_profiles.keys()[0] + template_name = tuple(matching_profiles.keys())[0] self.log.warning(( "More than one template profiles matched" " Family \"{}\" and Task: \"{}\"." From 70b1cd004a515f86fb3d7c097e86b0dd6827250c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 22 May 2020 14:15:01 +0200 Subject: [PATCH 71/71] don't use host name from pyblish's registered_host but from AVALON_APP environemnt --- pype/plugins/global/publish/extract_burnin.py | 2 +- pype/plugins/global/publish/extract_review.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index d0a5364945d..2eac38bac88 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -73,7 +73,7 @@ def process(self, instance): def main_process(self, instance): # TODO get these data from context - host_name = pyblish.api.registered_hosts()[-1] + host_name = os.environ["AVALON_APP"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 5b35e727ac6..228b4cd6f4c 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -68,7 +68,7 @@ def process(self, instance): instance.data["representations"].remove(repre) def main_process(self, instance): - host_name = pyblish.api.registered_hosts()[-1] + host_name = os.environ["AVALON_APP"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance)