From 840792a82c2c443dee77b4e8ec02ce6ec72b1e70 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 13 Oct 2022 19:14:48 +0200 Subject: [PATCH 1/6] added mechanism to define custom paths to ffmpeg and oiio tools and more detailed validation of them --- openpype/lib/vendor_bin_utils.py | 220 +++++++++++++++++++++++++++++-- 1 file changed, 212 insertions(+), 8 deletions(-) diff --git a/openpype/lib/vendor_bin_utils.py b/openpype/lib/vendor_bin_utils.py index e5ab2872a0e..31245d4ee43 100644 --- a/openpype/lib/vendor_bin_utils.py +++ b/openpype/lib/vendor_bin_utils.py @@ -1,10 +1,33 @@ import os import logging import platform +import subprocess log = logging.getLogger("Vendor utils") +class CachedToolPaths: + """Cache already used and discovered tools and their executables. + + Discovering path can take some time and can trigger subprocesses so it's + better to cache the paths on first get. + """ + + _cached_paths = {} + + @classmethod + def is_tool_cached(cls, tool): + return tool in cls._cached_paths + + @classmethod + def get_executable_path(cls, tool): + return cls._cached_paths.get(tool) + + @classmethod + def cache_executable_path(cls, tool, path): + cls._cached_paths[tool] = path + + def is_file_executable(filepath): """Filepath lead to executable file. @@ -98,6 +121,7 @@ def get_vendor_bin_path(bin_app): Returns: str: Path to vendorized binaries folder. """ + return os.path.join( os.environ["OPENPYPE_ROOT"], "vendor", @@ -107,6 +131,112 @@ def get_vendor_bin_path(bin_app): ) +def find_tool_in_custom_paths(paths, tool, validation_func=None): + """Find a tool executable in custom paths. + + Args: + paths (Iterable[str]): Iterable of paths where to look for tool. + tool (str): Name of tool (binary file) to find in passed paths. + validation_func (Function): Custom validation function of path. + Function must expect one argument which is path to executable. + If not passed only 'find_executable' is used to be able identify + if path is valid. + + Reuturns: + Union[str, None]: Path to validated executable or None if was not + found. + """ + + for path in paths: + # Skip empty strings + if not path: + continue + + # Handle cases when path is just an executable + # - it allows to use executable from PATH + # - basename must match 'tool' value (without extension) + extless_path, ext = os.path.splitext(path) + if extless_path == tool: + executable_path = find_executable(tool) + if executable_path and ( + validation_func is None + or validation_func(executable_path) + ): + return executable_path + continue + + # Normalize path because it should be a path and check if exists + normalized = os.path.normpath(path) + if not os.path.exists(normalized): + continue + + # Note: Path can be both file and directory + + # If path is a file validate it + if os.path.isfile(normalized): + basename, ext = os.path.splitext(os.path.basename(path)) + # Check if the filename has actually the sane bane as 'tool' + if basename == tool: + executable_path = find_executable(normalized) + if executable_path and ( + validation_func is None + or validation_func(executable_path) + ): + return executable_path + + # Check if path is a directory and look for tool inside the dir + if os.path.isdir(normalized): + executable_path = find_executable(os.path.join(normalized, tool)) + if executable_path and ( + validation_func is None + or validation_func(executable_path) + ): + return executable_path + return None + + +def _oiio_executable_validation(filepath): + """Validate oiio tool executable if can be executed. + + Validation has 2 steps. First is using 'find_executable' to fill possible + missing extension or fill directory then launch executable and validate + that it can be executed. For that is used '--help' argument which is fast + and does not need any other inputs. + + Any possible crash of missing libraries or invalid build should be catched. + + Main reason is to validate if executable can be executed on OS just running + which can be issue ob linux machines. + + Note: + It does not validate if the executable is really a oiio tool which + should be used. + + Args: + filepath (str): Path to executable. + + Returns: + bool: Filepath is valid executable. + """ + + filepath = find_executable(filepath) + if not filepath: + return False + + try: + proc = subprocess.Popen( + [filepath, "--help"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + proc.wait() + return proc.returncode == 0 + + except Exception: + pass + return False + + def get_oiio_tools_path(tool="oiiotool"): """Path to vendorized OpenImageIO tool executables. @@ -117,10 +247,67 @@ def get_oiio_tools_path(tool="oiiotool"): Default is "oiiotool". """ - oiio_dir = get_vendor_bin_path("oiio") - if platform.system().lower() == "linux": - oiio_dir = os.path.join(oiio_dir, "bin") - return find_executable(os.path.join(oiio_dir, tool)) + if CachedToolPaths.is_tool_cached(tool): + return CachedToolPaths.get_executable_path(tool) + + custom_paths_str = os.environ.get("OPENPYPE_CUSTOM_OIIO_PATHS") or "" + tool_executable_path = find_tool_in_custom_paths( + custom_paths_str.split(os.pathsep), + tool, + _oiio_executable_validation + ) + + if not tool_executable_path: + oiio_dir = get_vendor_bin_path("oiio") + if platform.system().lower() == "linux": + oiio_dir = os.path.join(oiio_dir, "bin") + default_path = os.path.join(oiio_dir, tool) + if _oiio_executable_validation(default_path): + tool_executable_path = default_path + + CachedToolPaths.cache_executable_path(tool, tool_executable_path) + return tool_executable_path + + +def _ffmpeg_executable_validation(filepath): + """Validate ffmpeg tool executable if can be executed. + + Validation has 2 steps. First is using 'find_executable' to fill possible + missing extension or fill directory then launch executable and validate + that it can be executed. For that is used '-version' argument which is fast + and does not need any other inputs. + + Any possible crash of missing libraries or invalid build should be catched. + + Main reason is to validate if executable can be executed on OS just running + which can be issue ob linux machines. + + Note: + It does not validate if the executable is really a ffmpeg tool. + + Args: + filepath (str): Path to executable. + + Returns: + bool: Filepath is valid executable. + """ + + filepath = find_executable(filepath) + if not filepath: + return False + + try: + proc = subprocess.Popen( + [filepath, "-version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + proc.wait() + return proc.returncode == 0 + + except Exception: + pass + return False def get_ffmpeg_tool_path(tool="ffmpeg"): @@ -133,10 +320,27 @@ def get_ffmpeg_tool_path(tool="ffmpeg"): Returns: str: Full path to ffmpeg executable. """ - ffmpeg_dir = get_vendor_bin_path("ffmpeg") - if platform.system().lower() == "windows": - ffmpeg_dir = os.path.join(ffmpeg_dir, "bin") - return find_executable(os.path.join(ffmpeg_dir, tool)) + + if CachedToolPaths.is_tool_cached(tool): + return CachedToolPaths.get_executable_path(tool) + + custom_paths_str = os.environ.get("OPENPYPE_CUSTOM_FFMPEG_PATHS") or "" + tool_executable_path = find_tool_in_custom_paths( + custom_paths_str.split(os.pathsep), + tool, + _ffmpeg_executable_validation + ) + + if not tool_executable_path: + ffmpeg_dir = get_vendor_bin_path("ffmpeg") + if platform.system().lower() == "windows": + ffmpeg_dir = os.path.join(ffmpeg_dir, "bin") + tool_path = find_executable(os.path.join(ffmpeg_dir, tool)) + if tool_path and _ffmpeg_executable_validation(tool_path): + tool_executable_path = tool_path + + CachedToolPaths.cache_executable_path(tool, tool_executable_path) + return tool_executable_path def is_oiio_supported(): From e3c2bb5a5e4d8a91508e0cd0db9ea2727424d6ed Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 14 Oct 2022 09:46:14 +0200 Subject: [PATCH 2/6] added one more last check for executable --- openpype/lib/vendor_bin_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/openpype/lib/vendor_bin_utils.py b/openpype/lib/vendor_bin_utils.py index 31245d4ee43..7b52341290d 100644 --- a/openpype/lib/vendor_bin_utils.py +++ b/openpype/lib/vendor_bin_utils.py @@ -265,6 +265,12 @@ def get_oiio_tools_path(tool="oiiotool"): if _oiio_executable_validation(default_path): tool_executable_path = default_path + # Look to PATH for the tool + if not tool_executable_path: + from_path = find_executable(tool) + if from_path and _oiio_executable_validation(from_path): + tool_executable_path = from_path + CachedToolPaths.cache_executable_path(tool, tool_executable_path) return tool_executable_path @@ -339,6 +345,12 @@ def get_ffmpeg_tool_path(tool="ffmpeg"): if tool_path and _ffmpeg_executable_validation(tool_path): tool_executable_path = tool_path + # Look to PATH for the tool + if not tool_executable_path: + from_path = find_executable(tool) + if from_path and _oiio_executable_validation(from_path): + tool_executable_path = from_path + CachedToolPaths.cache_executable_path(tool, tool_executable_path) return tool_executable_path From def2fc4bc81d3ec21f8a6cb59726456d7c3d4080 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 18 Oct 2022 14:51:54 +0200 Subject: [PATCH 3/6] removed '_CUSTOM' from env keys --- openpype/lib/vendor_bin_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/lib/vendor_bin_utils.py b/openpype/lib/vendor_bin_utils.py index 7b52341290d..eb7987c8a10 100644 --- a/openpype/lib/vendor_bin_utils.py +++ b/openpype/lib/vendor_bin_utils.py @@ -250,7 +250,7 @@ def get_oiio_tools_path(tool="oiiotool"): if CachedToolPaths.is_tool_cached(tool): return CachedToolPaths.get_executable_path(tool) - custom_paths_str = os.environ.get("OPENPYPE_CUSTOM_OIIO_PATHS") or "" + custom_paths_str = os.environ.get("OPENPYPE_OIIO_PATHS") or "" tool_executable_path = find_tool_in_custom_paths( custom_paths_str.split(os.pathsep), tool, @@ -330,7 +330,7 @@ def get_ffmpeg_tool_path(tool="ffmpeg"): if CachedToolPaths.is_tool_cached(tool): return CachedToolPaths.get_executable_path(tool) - custom_paths_str = os.environ.get("OPENPYPE_CUSTOM_FFMPEG_PATHS") or "" + custom_paths_str = os.environ.get("OPENPYPE_FFMPEG_PATHS") or "" tool_executable_path = find_tool_in_custom_paths( custom_paths_str.split(os.pathsep), tool, From 8e90bf73c9bb0ccbb46fedcc557118b2cfb8adb8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 18 Oct 2022 15:00:56 +0200 Subject: [PATCH 4/6] added information about ffmeg and oiio to documentation --- website/docs/admin_settings_system.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/website/docs/admin_settings_system.md b/website/docs/admin_settings_system.md index 8daba91db17..800ec1c8408 100644 --- a/website/docs/admin_settings_system.md +++ b/website/docs/admin_settings_system.md @@ -26,7 +26,14 @@ as a naive barier to prevent artists from accidental setting changes. **`Disk mapping`** - Platform dependent configuration for mapping of virtual disk(s) on an artist's OpenPype machines before OP starts up. Uses `subst` command, if configured volume character in `Destination` field already exists, no re-mapping is done for that character(volume). + +### FFmpeg and OpenImageIO tools + +We bundle FFmpeg and OpenImageIO tools with OpenPype build for Windows and Linux builds. For MacOs support or to use different build is it possible to set environment variables `OPENPYPE_FFMPEG_PATHS` and `OPENPYPE_OIIO_PATHS` in system settings. Both should lead to directory where tool executables are located. Multiple paths are supported. + + ### OpenPype deployment control + **`Versions Repository`** - Location where automatic update mechanism searches for zip files with OpenPype update packages. To read more about preparing OpenPype for automatic updates go to [Admin Distribute docs](admin_distribute.md#2-openpype-codebase) From 952fd6c15c6f60a062bd3e14e0bc6df5dd7b33ee Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 18 Oct 2022 15:01:43 +0200 Subject: [PATCH 5/6] removed unnecessary lines --- website/docs/admin_settings_system.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/website/docs/admin_settings_system.md b/website/docs/admin_settings_system.md index 800ec1c8408..cef4571c84b 100644 --- a/website/docs/admin_settings_system.md +++ b/website/docs/admin_settings_system.md @@ -26,14 +26,10 @@ as a naive barier to prevent artists from accidental setting changes. **`Disk mapping`** - Platform dependent configuration for mapping of virtual disk(s) on an artist's OpenPype machines before OP starts up. Uses `subst` command, if configured volume character in `Destination` field already exists, no re-mapping is done for that character(volume). - ### FFmpeg and OpenImageIO tools - We bundle FFmpeg and OpenImageIO tools with OpenPype build for Windows and Linux builds. For MacOs support or to use different build is it possible to set environment variables `OPENPYPE_FFMPEG_PATHS` and `OPENPYPE_OIIO_PATHS` in system settings. Both should lead to directory where tool executables are located. Multiple paths are supported. - ### OpenPype deployment control - **`Versions Repository`** - Location where automatic update mechanism searches for zip files with OpenPype update packages. To read more about preparing OpenPype for automatic updates go to [Admin Distribute docs](admin_distribute.md#2-openpype-codebase) From e9db3dbadce7bee241608f010785a3b14ce95641 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 18 Oct 2022 15:07:56 +0200 Subject: [PATCH 6/6] rephrase sentence --- website/docs/admin_settings_system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/admin_settings_system.md b/website/docs/admin_settings_system.md index cef4571c84b..66715e72887 100644 --- a/website/docs/admin_settings_system.md +++ b/website/docs/admin_settings_system.md @@ -27,7 +27,7 @@ as a naive barier to prevent artists from accidental setting changes. Uses `subst` command, if configured volume character in `Destination` field already exists, no re-mapping is done for that character(volume). ### FFmpeg and OpenImageIO tools -We bundle FFmpeg and OpenImageIO tools with OpenPype build for Windows and Linux builds. For MacOs support or to use different build is it possible to set environment variables `OPENPYPE_FFMPEG_PATHS` and `OPENPYPE_OIIO_PATHS` in system settings. Both should lead to directory where tool executables are located. Multiple paths are supported. +We bundle FFmpeg tools for all platforms and OpenImageIO tools for Windows and Linux. By default are used bundled tools but it is possible to set environment variables `OPENPYPE_FFMPEG_PATHS` and `OPENPYPE_OIIO_PATHS` in system settings environments to look for them in different directory e.g. for different linux distributions or to add oiio support for MacOs. Values of both environment variables should lead to directory where tool executables are located (multiple paths are supported). ### OpenPype deployment control **`Versions Repository`** - Location where automatic update mechanism searches for zip files with