Merge pull request #795 from pypeclub/dwaa-support-on-master

DWAA/DWAB support on windows

pype/lib/__init__.py

@@ -29,7 +29,11 @@
  filter_pyblish_plugins,
  source_hash,
  get_unique_layer_name,
- get_background_layers
+ get_background_layers,
+ oiio_supported,
+ decompress,
+ get_decompress_dir,
+ should_decompress
 )
 
 from .path_tools import (
@@ -64,6 +68,10 @@
  "filter_pyblish_plugins",
  "get_unique_layer_name",
  "get_background_layers",
+ "oiio_supported",
+ "decompress",
+ "get_decompress_dir",
+ "should_decompress",
 
  "version_up",
  "get_version_from_path",

pype/lib/plugin_tools.py

@@ -5,6 +5,8 @@
 import logging
 import re
 import json
+import pype.api
+import tempfile
 
 from ..api import config
 
@@ -134,3 +136,115 @@ def get_background_layers(file_url):
  layer.get("filename")).
  replace("\\", "/"))
  return layers
+
+
+def oiio_supported():
+ """
+ Checks if oiiotool is configured for this platform.
+
+ Expects full path to executable.
+
+ 'should_decompress' will throw exception if configured,
+ but not present or not working.
+ Returns:
+ (bool)
+ """
+ oiio_path = os.getenv("PYPE_OIIO_PATH", "")
+ if not oiio_path or not os.path.exists(oiio_path):
+ log.debug("OIIOTool is not configured or not present at {}".
+ format(oiio_path))
+ return False
+
+ return True
+
+
+def decompress(target_dir, file_url,
+ input_frame_start=None, input_frame_end=None, log=None):
+ """
+ Decompresses DWAA 'file_url' .exr to 'target_dir'.
+
+ Creates uncompressed files in 'target_dir', they need to be cleaned.
+
+ File url could be for single file or for a sequence, in that case
+ %0Xd will be as a placeholder for frame number AND input_frame* will
+ be filled.
+ In that case single oiio command with '--frames' will be triggered for
+ all frames, this should be faster then looping and running sequentially
+
+ Args:
+ target_dir (str): extended from stagingDir
+ file_url (str): full urls to source file (with or without %0Xd)
+ input_frame_start (int) (optional): first frame
+ input_frame_end (int) (optional): last frame
+ log (Logger) (optional): pype logger
+ """
+ is_sequence = input_frame_start is not None and \
+ input_frame_end is not None and \
+ (int(input_frame_end) > int(input_frame_start))
+
+ oiio_cmd = []
+ oiio_cmd.append(os.getenv("PYPE_OIIO_PATH"))
+
+ oiio_cmd.append("--compression none")
+
+ base_file_name = os.path.basename(file_url)
+ oiio_cmd.append(file_url)
+
+ if is_sequence:
+ oiio_cmd.append("--frames {}-{}".format(input_frame_start,
+ input_frame_end))
+
+ oiio_cmd.append("-o")
+ oiio_cmd.append(os.path.join(target_dir, base_file_name))
+
+ subprocess_exr = " ".join(oiio_cmd)
+
+ if not log:
+ log = logging.getLogger(__name__)
+
+ log.debug("Decompressing {}".format(subprocess_exr))
+ pype.api.subprocess(
+ subprocess_exr, shell=True, logger=log
+ )
+
+
+def get_decompress_dir():
+ """
+ Creates temporary folder for decompressing.
+ Its local, in case of farm it is 'local' to the farm machine.
+
+ Should be much faster, needs to be cleaned up later.
+ """
+ return os.path.normpath(
+ tempfile.mkdtemp(prefix="pyblish_tmp_")
+ )
+
+
+def should_decompress(file_url):
+ """
+ Tests that 'file_url' is compressed with DWAA.
+
+ Uses 'oiio_supported' to check that OIIO tool is available for this
+ platform.
+
+ Shouldn't throw exception as oiiotool is guarded by check function.
+ Currently implemented this way as there is no support for Mac and Linux
+ In the future, it should be more strict and throws exception on
+ misconfiguration.
+
+ Args:
+ file_url (str): path to rendered file (in sequence it would be
+ first file, if that compressed it is expected that whole seq
+ will be too)
+ Returns:
+ (bool): 'file_url' is DWAA compressed and should be decompressed
+ and we can decompress (oiiotool supported)
+ """
+ if oiio_supported():
+ output = pype.api.subprocess([
+ os.getenv("PYPE_OIIO_PATH"),
+ "--info", "-v", file_url])
+ return "compression: \"dwaa\"" in output or \
+ "compression: \"dwab\"" in output
+
+ return False

pype/plugins/global/publish/extract_burnin.py

@@ -6,6 +6,9 @@
 
 import pype.api
 import pyblish
+from pype.lib import should_decompress, \
+ get_decompress_dir, decompress
+import shutil
 
 
 class ExtractBurnin(pype.api.Extractor):
@@ -28,7 +31,8 @@ class ExtractBurnin(pype.api.Extractor):
  "premiere",
  "standalonepublisher",
  "harmony",
- "fusion"
+ "fusion",
+ "aftereffects"
  ]
  optional = True
 
@@ -212,6 +216,26 @@ def main_process(self, instance):
  # Prepare paths and files for process.
  self.input_output_paths(new_repre, temp_data, filename_suffix)
 
+ decompressed_dir = ''
+ full_input_path = temp_data["full_input_path"]
+ do_decompress = should_decompress(full_input_path)
+ if do_decompress:
+ decompressed_dir = get_decompress_dir()
+
+ decompress(
+ decompressed_dir,
+ full_input_path,
+ temp_data["frame_start"],
+ temp_data["frame_end"],
+ self.log
+ )
+
+ # input path changed, 'decompressed' added
+ input_file = os.path.basename(full_input_path)
+ temp_data["full_input_path"] = os.path.join(
+ decompressed_dir,
+ input_file)
+
  # Data for burnin script
  script_data = {
  "input": temp_data["full_input_path"],
@@ -271,6 +295,9 @@ def main_process(self, instance):
  os.remove(filepath)
  self.log.debug("Removed: \"{}\"".format(filepath))
 
+ if do_decompress and os.path.exists(decompressed_dir):
+ shutil.rmtree(decompressed_dir)
+
  def prepare_basic_data(self, instance):
  """Pick data from instance for processing and for burnin strings.
 

pype/plugins/global/publish/extract_jpeg.py

@@ -3,6 +3,9 @@
 import pyblish.api
 import pype.api
 import pype.lib
+from pype.lib import should_decompress, \
+ get_decompress_dir, decompress
+import shutil
 
 
 class ExtractJpegEXR(pyblish.api.InstancePlugin):
@@ -22,7 +25,8 @@ def process(self, instance):
  if 'crypto' in instance.data['subset']:
  return
 
- # ffmpeg doesn't support multipart exrs
+ do_decompress = False
+ # ffmpeg doesn't support multipart exrs, use oiiotool if available
  if instance.data.get("multipartExr") is True:
  return
 
@@ -36,10 +40,6 @@ def process(self, instance):
  # filter out mov and img sequences
  representations_new = representations[:]
 
- if instance.data.get("multipartExr"):
- # ffmpeg doesn't support multipart exrs
- return
-
  for repre in representations:
  tags = repre.get("tags", [])
  self.log.debug(repre)
@@ -60,6 +60,19 @@ def process(self, instance):
  full_input_path = os.path.join(stagingdir, input_file)
  self.log.info("input {}".format(full_input_path))
 
+ decompressed_dir = ''
+ do_decompress = should_decompress(full_input_path)
+ if do_decompress:
+ decompressed_dir = get_decompress_dir()
+
+ decompress(
+ decompressed_dir,
+ full_input_path)
+ # input path changed, 'decompressed' added
+ full_input_path = os.path.join(
+ decompressed_dir,
+ input_file)
+
  filename = os.path.splitext(input_file)[0]
  if not filename.endswith('.'):
  filename += "."
@@ -93,7 +106,14 @@ def process(self, instance):
 
  # run subprocess
  self.log.debug("{}".format(subprocess_jpeg))
- pype.api.subprocess(subprocess_jpeg, shell=True)
+ try: # temporary until oiiotool is supported cross platform
+ pype.api.subprocess(subprocess_jpeg, shell=True)
+ except RuntimeError as exp:
+ if "Compression" in str(exp):
+ self.log.debug("Unsupported compression on input files. " +
+ "Skipping!!!")
+ return
+ raise
 
  if "representations" not in instance.data:
  instance.data["representations"] = []
@@ -111,4 +131,7 @@ def process(self, instance):
  self.log.debug("Adding: {}".format(representation))
  representations_new.append(representation)
 
+ if do_decompress and os.path.exists(decompressed_dir):
+ shutil.rmtree(decompressed_dir)
+
  instance.data["representations"] = representations_new