General: Custom paths to ffmpeg and OpenImageIO tools

openpype/lib/vendor_bin_utils.py

@@ -1,10 +1,33 @@
 import os
 import logging
 import platform
+import subprocess
 
 log = logging.getLogger("Vendor utils")
 
 
+class CachedToolPaths:
+ """Cache already used and discovered tools and their executables.
+
+ Discovering path can take some time and can trigger subprocesses so it's
+ better to cache the paths on first get.
+ """
+
+ _cached_paths = {}
+
+ @classmethod
+ def is_tool_cached(cls, tool):
+ return tool in cls._cached_paths
+
+ @classmethod
+ def get_executable_path(cls, tool):
+ return cls._cached_paths.get(tool)
+
+ @classmethod
+ def cache_executable_path(cls, tool, path):
+ cls._cached_paths[tool] = path
+
+
 def is_file_executable(filepath):
  """Filepath lead to executable file.
 
@@ -98,6 +121,7 @@ def get_vendor_bin_path(bin_app):
  Returns:
  str: Path to vendorized binaries folder.
  """
+
  return os.path.join(
  os.environ["OPENPYPE_ROOT"],
  "vendor",
@@ -107,6 +131,112 @@ def get_vendor_bin_path(bin_app):
  )
 
 
+def find_tool_in_custom_paths(paths, tool, validation_func=None):
+ """Find a tool executable in custom paths.
+
+ Args:
+ paths (Iterable[str]): Iterable of paths where to look for tool.
+ tool (str): Name of tool (binary file) to find in passed paths.
+ validation_func (Function): Custom validation function of path.
+ Function must expect one argument which is path to executable.
+ If not passed only 'find_executable' is used to be able identify
+ if path is valid.
+
+ Reuturns:
+ Union[str, None]: Path to validated executable or None if was not
+ found.
+ """
+
+ for path in paths:
+ # Skip empty strings
+ if not path:
+ continue
+
+ # Handle cases when path is just an executable
+ # - it allows to use executable from PATH
+ # - basename must match 'tool' value (without extension)
+ extless_path, ext = os.path.splitext(path)
+ if extless_path == tool:
+ executable_path = find_executable(tool)
+ if executable_path and (
+ validation_func is None
+ or validation_func(executable_path)
+ ):
+ return executable_path
+ continue
+
+ # Normalize path because it should be a path and check if exists
+ normalized = os.path.normpath(path)
+ if not os.path.exists(normalized):
+ continue
+
+ # Note: Path can be both file and directory
+
+ # If path is a file validate it
+ if os.path.isfile(normalized):
+ basename, ext = os.path.splitext(os.path.basename(path))
+ # Check if the filename has actually the sane bane as 'tool'
+ if basename == tool:
+ executable_path = find_executable(normalized)
+ if executable_path and (
+ validation_func is None
+ or validation_func(executable_path)
+ ):
+ return executable_path
+
+ # Check if path is a directory and look for tool inside the dir
+ if os.path.isdir(normalized):
+ executable_path = find_executable(os.path.join(normalized, tool))
+ if executable_path and (
+ validation_func is None
+ or validation_func(executable_path)
+ ):
+ return executable_path
+ return None
+
+
+def _oiio_executable_validation(filepath):
+ """Validate oiio tool executable if can be executed.
+
+ Validation has 2 steps. First is using 'find_executable' to fill possible
+ missing extension or fill directory then launch executable and validate
+ that it can be executed. For that is used '--help' argument which is fast
+ and does not need any other inputs.
+
+ Any possible crash of missing libraries or invalid build should be catched.
+
+ Main reason is to validate if executable can be executed on OS just running
+ which can be issue ob linux machines.
+
+ Note:
+ It does not validate if the executable is really a oiio tool which
+ should be used.
+
+ Args:
+ filepath (str): Path to executable.
+
+ Returns:
+ bool: Filepath is valid executable.
+ """
+
+ filepath = find_executable(filepath)
+ if not filepath:
+ return False
+
+ try:
+ proc = subprocess.Popen(
+ [filepath, "--help"],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ proc.wait()
+ return proc.returncode == 0
+
+ except Exception:
+ pass
+ return False
+
+
 def get_oiio_tools_path(tool="oiiotool"):
  """Path to vendorized OpenImageIO tool executables.
 
@@ -117,10 +247,73 @@ def get_oiio_tools_path(tool="oiiotool"):
  Default is "oiiotool".
  """
 
- oiio_dir = get_vendor_bin_path("oiio")
- if platform.system().lower() == "linux":
- oiio_dir = os.path.join(oiio_dir, "bin")
- return find_executable(os.path.join(oiio_dir, tool))
+ if CachedToolPaths.is_tool_cached(tool):
+ return CachedToolPaths.get_executable_path(tool)
+
+ custom_paths_str = os.environ.get("OPENPYPE_OIIO_PATHS") or ""
+ tool_executable_path = find_tool_in_custom_paths(
+ custom_paths_str.split(os.pathsep),
+ tool,
+ _oiio_executable_validation
+ )
+
+ if not tool_executable_path:
+ oiio_dir = get_vendor_bin_path("oiio")
+ if platform.system().lower() == "linux":
+ oiio_dir = os.path.join(oiio_dir, "bin")
+ default_path = os.path.join(oiio_dir, tool)
+ if _oiio_executable_validation(default_path):
+ tool_executable_path = default_path
+
+ # Look to PATH for the tool
+ if not tool_executable_path:
+ from_path = find_executable(tool)
+ if from_path and _oiio_executable_validation(from_path):
+ tool_executable_path = from_path
+
+ CachedToolPaths.cache_executable_path(tool, tool_executable_path)
+ return tool_executable_path
+
+
+def _ffmpeg_executable_validation(filepath):
+ """Validate ffmpeg tool executable if can be executed.
+
+ Validation has 2 steps. First is using 'find_executable' to fill possible
+ missing extension or fill directory then launch executable and validate
+ that it can be executed. For that is used '-version' argument which is fast
+ and does not need any other inputs.
+
+ Any possible crash of missing libraries or invalid build should be catched.
+
+ Main reason is to validate if executable can be executed on OS just running
+ which can be issue ob linux machines.
+
+ Note:
+ It does not validate if the executable is really a ffmpeg tool.
+
+ Args:
+ filepath (str): Path to executable.
+
+ Returns:
+ bool: Filepath is valid executable.
+ """
+
+ filepath = find_executable(filepath)
+ if not filepath:
+ return False
+
+ try:
+ proc = subprocess.Popen(
+ [filepath, "-version"],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ proc.wait()
+ return proc.returncode == 0
+
+ except Exception:
+ pass
+ return False
 
 
 def get_ffmpeg_tool_path(tool="ffmpeg"):
@@ -133,10 +326,33 @@ def get_ffmpeg_tool_path(tool="ffmpeg"):
  Returns:
  str: Full path to ffmpeg executable.
  """
- ffmpeg_dir = get_vendor_bin_path("ffmpeg")
- if platform.system().lower() == "windows":
- ffmpeg_dir = os.path.join(ffmpeg_dir, "bin")
- return find_executable(os.path.join(ffmpeg_dir, tool))
+
+ if CachedToolPaths.is_tool_cached(tool):
+ return CachedToolPaths.get_executable_path(tool)
+
+ custom_paths_str = os.environ.get("OPENPYPE_FFMPEG_PATHS") or ""
+ tool_executable_path = find_tool_in_custom_paths(
+ custom_paths_str.split(os.pathsep),
+ tool,
+ _ffmpeg_executable_validation
+ )
+
+ if not tool_executable_path:
+ ffmpeg_dir = get_vendor_bin_path("ffmpeg")
+ if platform.system().lower() == "windows":
+ ffmpeg_dir = os.path.join(ffmpeg_dir, "bin")
+ tool_path = find_executable(os.path.join(ffmpeg_dir, tool))
+ if tool_path and _ffmpeg_executable_validation(tool_path):
+ tool_executable_path = tool_path
+
+ # Look to PATH for the tool
+ if not tool_executable_path:
+ from_path = find_executable(tool)
+ if from_path and _oiio_executable_validation(from_path):
+ tool_executable_path = from_path
+
+ CachedToolPaths.cache_executable_path(tool, tool_executable_path)
+ return tool_executable_path
 
 
 def is_oiio_supported():

website/docs/admin_settings_system.md

@@ -26,6 +26,9 @@ as a naive barier to prevent artists from accidental setting changes.
 **`Disk mapping`** - Platform dependent configuration for mapping of virtual disk(s) on an artist's OpenPype machines before OP starts up. 
 Uses `subst` command, if configured volume character in `Destination` field already exists, no re-mapping is done for that character(volume).
 
+### FFmpeg and OpenImageIO tools
+We bundle FFmpeg tools for all platforms and OpenImageIO tools for Windows and Linux. By default are used bundled tools but it is possible to set environment variables `OPENPYPE_FFMPEG_PATHS` and `OPENPYPE_OIIO_PATHS` in system settings environments to look for them in different directory e.g. for different linux distributions or to add oiio support for MacOs. Values of both environment variables should lead to directory where tool executables are located (multiple paths are supported).
+
 ### OpenPype deployment control
 **`Versions Repository`** - Location where automatic update mechanism searches for zip files with
 OpenPype update packages. To read more about preparing OpenPype for automatic updates go to [Admin Distribute docs](admin_distribute.md#2-openpype-codebase)