From 0dd46fe51329b15dd6a73e68534904d11d194587 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 20 Apr 2022 14:40:22 +0200 Subject: [PATCH 1/6] added new function convert_input_paths_for_ffmpeg which converts list of input paths to output dir keeping the source filenames --- openpype/lib/transcoding.py | 122 ++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index c2fecf6628c..f379f81decc 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -516,6 +516,128 @@ def convert_for_ffmpeg( run_subprocess(oiio_cmd, logger=logger) +def convert_input_paths_for_ffmpeg( + input_paths, + output_dir, + logger=None +): + """Contert source file to format supported in ffmpeg. + + Currently can convert only exrs. The input filepaths should be files + with same type. Information about input is loaded only from first found + file. + + Filenames of input files are kept so make sure that output directory + is not the same directory as input files have. + + Args: + input_paths (str): Paths that should be converted. It is expected that + contains single file or image sequence of samy type. + output_dir (str): Path to directory where output will be rendered. + Must not be same as input's directory. + logger (logging.Logger): Logger used for logging. + + Raises: + ValueError: If input filepath has extension not supported by function. + Currently is supported only ".exr" extension. + """ + if logger is None: + logger = logging.getLogger(__name__) + + first_input_path = input_paths[0] + ext = os.path.splitext(first_input_path)[1].lower() + if ext != ".exr": + raise ValueError(( + "Function 'convert_for_ffmpeg' currently support only" + " \".exr\" extension. Got \"{}\"." + ).format(ext)) + + input_info = get_oiio_info_for_input(first_input_path) + + # Change compression only if source compression is "dwaa" or "dwab" + # - they're not supported in ffmpeg + compression = input_info["attribs"].get("compression") + if compression in ("dwaa", "dwab"): + compression = "none" + + # Collect channels to export + channel_names = input_info["channelnames"] + review_channels = get_convert_rgb_channels(channel_names) + if review_channels is None: + raise ValueError( + "Couldn't find channels that can be used for conversion." + ) + + red, green, blue, alpha = review_channels + input_channels = [red, green, blue] + channels_arg = "R={},G={},B={}".format(red, green, blue) + if alpha is not None: + channels_arg += ",A={}".format(alpha) + input_channels.append(alpha) + input_channels_str = ",".join(input_channels) + + for input_path in input_paths: + # Prepare subprocess arguments + oiio_cmd = [ + get_oiio_tools_path(), + + # Don't add any additional attributes + "--nosoftwareattrib", + ] + # Add input compression if available + if compression: + oiio_cmd.extend(["--compression", compression]) + + oiio_cmd.extend([ + # Tell oiiotool which channels should be loaded + # - other channels are not loaded to memory so helps to + # avoid memory leak issues + "-i:ch={}".format(input_channels_str), input_path, + # Tell oiiotool which channels should be put to top stack + # (and output) + "--ch", channels_arg + ]) + + for attr_name, attr_value in input_info["attribs"].items(): + if not isinstance(attr_value, str): + continue + + # Remove attributes that have string value longer than allowed length + # for ffmpeg or when containt unallowed symbols + erase_reason = "Missing reason" + erase_attribute = False + if len(attr_value) > MAX_FFMPEG_STRING_LEN: + erase_reason = "has too long value ({} chars).".format( + len(attr_value) + ) + + if erase_attribute: + for char in NOT_ALLOWED_FFMPEG_CHARS: + if char in attr_value: + erase_attribute = True + erase_reason = ( + "contains unsupported character \"{}\"." + ).format(char) + break + + if erase_attribute: + # Set attribute to empty string + logger.info(( + "Removed attribute \"{}\" from metadata because {}." + ).format(attr_name, erase_reason)) + oiio_cmd.extend(["--eraseattrib", attr_name]) + + # Add last argument - path to output + base_filename = os.path.basename(first_input_path) + output_path = os.path.join(output_dir, base_filename) + oiio_cmd.extend([ + "-o", output_path + ]) + + logger.debug("Conversion command: {}".format(" ".join(oiio_cmd))) + run_subprocess(oiio_cmd, logger=logger) + + # FFMPEG functions def get_ffprobe_data(path_to_file, logger=None): """Load data about entered filepath via ffprobe. From 91d2eb7355a2934e1afeb2371cb9bb97b501f7ae Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 20 Apr 2022 14:41:03 +0200 Subject: [PATCH 2/6] replace convert_for_ffmpeg with new function --- .../plugins/publish/extract_thumbnail.py | 8 +++----- openpype/lib/__init__.py | 2 ++ openpype/plugins/publish/extract_burnin.py | 13 ++++++++----- openpype/plugins/publish/extract_jpeg_exr.py | 8 +++----- openpype/plugins/publish/extract_review.py | 18 +++++++++--------- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py b/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py index cb6ed8481cf..a56521891bf 100644 --- a/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py +++ b/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py @@ -8,7 +8,7 @@ run_subprocess, get_transcode_temp_directory, - convert_for_ffmpeg, + convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg ) @@ -59,11 +59,9 @@ def process(self, instance): if do_convert: convert_dir = get_transcode_temp_directory() filename = os.path.basename(full_input_path) - convert_for_ffmpeg( - full_input_path, + convert_input_paths_for_ffmpeg( + [full_input_path], convert_dir, - None, - None, self.log ) full_input_path = os.path.join(convert_dir, filename) diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index b57e469f5ba..29719b63bd3 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -105,6 +105,7 @@ get_transcode_temp_directory, should_convert_for_ffmpeg, convert_for_ffmpeg, + convert_input_paths_for_ffmpeg, get_ffprobe_data, get_ffprobe_streams, get_ffmpeg_codec_args, @@ -276,6 +277,7 @@ "get_transcode_temp_directory", "should_convert_for_ffmpeg", "convert_for_ffmpeg", + "convert_input_paths_for_ffmpeg", "get_ffprobe_data", "get_ffprobe_streams", "get_ffmpeg_codec_args", diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 41c84103a61..544c763b52f 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -16,7 +16,7 @@ run_openpype_process, get_transcode_temp_directory, - convert_for_ffmpeg, + convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg, CREATE_NO_WINDOW @@ -187,8 +187,13 @@ def main_process(self, instance): repre_files = repre["files"] if isinstance(repre_files, (tuple, list)): filename = repre_files[0] + src_filepaths = [ + os.path.join(src_repre_staging_dir, filename) + for filename in repre_files + ] else: filename = repre_files + src_filepaths = [os.path.join(src_repre_staging_dir, filename)] first_input_path = os.path.join(src_repre_staging_dir, filename) # Determine if representation requires pre conversion for ffmpeg @@ -209,11 +214,9 @@ def main_process(self, instance): new_staging_dir = get_transcode_temp_directory() repre["stagingDir"] = new_staging_dir - convert_for_ffmpeg( - first_input_path, + convert_input_paths_for_ffmpeg( + src_filepaths, new_staging_dir, - _temp_data["frameStart"], - _temp_data["frameEnd"], self.log ) diff --git a/openpype/plugins/publish/extract_jpeg_exr.py b/openpype/plugins/publish/extract_jpeg_exr.py index 468ed961998..d6d68540926 100644 --- a/openpype/plugins/publish/extract_jpeg_exr.py +++ b/openpype/plugins/publish/extract_jpeg_exr.py @@ -8,7 +8,7 @@ path_to_subprocess_arg, get_transcode_temp_directory, - convert_for_ffmpeg, + convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg ) @@ -79,11 +79,9 @@ def process(self, instance): if do_convert: convert_dir = get_transcode_temp_directory() filename = os.path.basename(full_input_path) - convert_for_ffmpeg( - full_input_path, + convert_input_paths_for_ffmpeg( + [full_input_path], convert_dir, - None, - None, self.log ) full_input_path = os.path.join(convert_dir, filename) diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index d569d82762f..f2473839d92 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -18,7 +18,7 @@ path_to_subprocess_arg, should_convert_for_ffmpeg, - convert_for_ffmpeg, + convert_input_paths_for_ffmpeg, get_transcode_temp_directory ) import speedcopy @@ -194,16 +194,20 @@ def main_process(self, instance): src_repre_staging_dir = repre["stagingDir"] # Receive filepath to first file in representation first_input_path = None + input_filepaths = [] if not self.input_is_sequence(repre): first_input_path = os.path.join( src_repre_staging_dir, repre["files"] ) + input_filepaths.append(first_input_path) else: for filename in repre["files"]: - first_input_path = os.path.join( + filepath = os.path.join( src_repre_staging_dir, filename ) - break + input_filepaths.append(filepath) + if first_input_path is None: + first_input_path = filepath # Skip if file is not set if first_input_path is None: @@ -230,13 +234,9 @@ def main_process(self, instance): new_staging_dir = get_transcode_temp_directory() repre["stagingDir"] = new_staging_dir - frame_start = instance.data["frameStart"] - frame_end = instance.data["frameEnd"] - convert_for_ffmpeg( - first_input_path, + convert_input_paths_for_ffmpeg( + input_filepaths, new_staging_dir, - frame_start, - frame_end, self.log ) From d9d772db72f8e085817d7e93b267ea8f70d75399 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 20 Apr 2022 14:41:15 +0200 Subject: [PATCH 3/6] added deprecation warning to convert_to_ffmpeg --- openpype/lib/transcoding.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index f379f81decc..b75ae2baf1e 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -409,6 +409,12 @@ def convert_for_ffmpeg( if logger is None: logger = logging.getLogger(__name__) + logger.warning(( + "DEPRECATED: 'openpype.lib.transcoding.convert_for_ffmpeg' is" + " deprecated function of conversion for FFMpeg. Please replace usage" + " with 'openpype.lib.transcoding.convert_input_paths_for_ffmpeg'" + )) + ext = os.path.splitext(first_input_path)[1].lower() if ext != ".exr": raise ValueError(( From 7deed5e74d8124b6f1d52cc45d1c9c06e4e0bd76 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 20 Apr 2022 15:28:15 +0200 Subject: [PATCH 4/6] added few more comments --- openpype/lib/transcoding.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index b75ae2baf1e..7cc345fcc0c 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -382,6 +382,11 @@ def should_convert_for_ffmpeg(src_filepath): return False +# Deprecated since 2022 4 20 +# - Reason - Doesn't convert sequences right way: Can't handle gaps, reuse +# first frame for all frames and changes filenames when input +# is sequence. +# - use 'convert_input_paths_for_ffmpeg' instead def convert_for_ffmpeg( first_input_path, output_dir, @@ -535,6 +540,8 @@ def convert_input_paths_for_ffmpeg( Filenames of input files are kept so make sure that output directory is not the same directory as input files have. + - This way it can handle gaps and can keep input filenames without handling + frame template Args: input_paths (str): Paths that should be converted. It is expected that From 52a1450df2a4c9f6604899add111670aafe7bd79 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 20 Apr 2022 15:28:49 +0200 Subject: [PATCH 5/6] fix line length --- openpype/lib/transcoding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index 7cc345fcc0c..cdfe240e68d 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -615,8 +615,8 @@ def convert_input_paths_for_ffmpeg( if not isinstance(attr_value, str): continue - # Remove attributes that have string value longer than allowed length - # for ffmpeg or when containt unallowed symbols + # Remove attributes that have string value longer than allowed + # length for ffmpeg or when containt unallowed symbols erase_reason = "Missing reason" erase_attribute = False if len(attr_value) > MAX_FFMPEG_STRING_LEN: From 5142cd613935e3e6b9c514c4a08371dad2555009 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 20 Apr 2022 19:07:56 +0200 Subject: [PATCH 6/6] fix keeping of filenames --- openpype/lib/transcoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index cdfe240e68d..fcec5d42169 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -641,7 +641,7 @@ def convert_input_paths_for_ffmpeg( oiio_cmd.extend(["--eraseattrib", attr_name]) # Add last argument - path to output - base_filename = os.path.basename(first_input_path) + base_filename = os.path.basename(input_path) output_path = os.path.join(output_dir, base_filename) oiio_cmd.extend([ "-o", output_path