From 37e88288b4f667d8151664a1b7f28e64ad944482 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 7 Sep 2020 14:01:34 +0200 Subject: [PATCH 1/7] tag instance for scanline conversion using oiio tools --- .../global/publish/extract_scanline_exr.py | 85 +++++++++++++++++++ .../global/publish/submit_publish_job.py | 7 +- pype/plugins/maya/create/create_render.py | 1 + 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 pype/plugins/global/publish/extract_scanline_exr.py diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py new file mode 100644 index 00000000000..e1ca0e7c1c3 --- /dev/null +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +"""Convert tiled exrs to scanline if needed.""" +import os +import copy + +import pyblish.api +import pype.api +import pype.lib + + +class ExtractScanlineExr(pyblish.api.InstancePlugin): + """Convert tiled EXRs to scanline using OIIO tool.""" + + label = "Extract Scanline EXR" + hosts = ["shell"] + order = pyblish.api.ExtractorOrder + families = ["imagesequence", "render", "render2d", "source"] + + def process(self, instance): + """Plugin entry point.""" + anatomy = instance.context.data['anatomy'] + + # get representation and loop them + representations = instance.data["representations"] + representations_new = [] + + for repre in representations: + tags = repre.get("tags", []) + if "toScanline" not in tags: + continue + + # run only on exrs + if repre.get("ext") != "exr": + continue + + if not isinstance(repre['files'], (list, tuple)): + input_files = [repre['files']] + self.log.info("We have a sequence.") + else: + input_files = repre['files'] + self.log.info("We have a single frame") + + stagingdir = os.path.normpath(repre.get("stagingDir")) + stagingdir = anatomy.fill_root(stagingdir) + + oiio_tool_path = os.getenv("PYPE_OIIO_PATH", "") + + new_files = [] + for file in input_files: + + oiio_cmd = [] + oiio_cmd.append(oiio_tool_path) + oiio_cmd.append( + os.path.join(stagingdir, file) + ) + oiio_cmd.append("--scanline") + oiio_cmd.append("-o") + new_file = f"_scanline_{file}" + new_files.append(new_file) + oiio_cmd.append(stagingdir, new_file) + + subprocess_exr = " ".join(oiio_cmd) + self.log.info(f"running: {subprocess_exr}") + pype.api.subprocess(subprocess_exr) + + # raise error if there is no ouptput + if not os.path.exists(os.path.join(stagingdir, new_file)): + self.log.error( + f"File {new_file} was not produced by oiio tool!") + raise AssertionError("OIIO tool conversion failed") + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = copy.deepcopy(repre) + + representation['name'] = 'scanline exr' + representation['files'] = new_files if len(new_files) > 1 else new_files[0] # noqa: E501 + representation['tags'].remove("toScanline") + + # add representation + self.log.debug("Adding: {}".format(representation)) + representations_new.append(representation) + + instance.data["representations"] = representations_new diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index ab5d6cf9b2e..2d6d1efff83 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -193,7 +193,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "slate": ["slateFrame"], "review": ["lutPath"], "render2d": ["bakeScriptPath", "bakeRenderPath", - "bakeWriteNodeName", "version"] + "bakeWriteNodeName", "version"], + "renderlayer": ["convertToScanline"] } # list of family names to transfer to new family if present @@ -491,6 +492,10 @@ def _create_instances_for_aov(self, instance_data, exp_files): "tags": ["review"] if preview else [] } + # support conversion from tiled to scanline + if instance_data.get("convertToScanline"): + rep["tags"].append("toScanline") + # poor man exclusion if ext in self.skip_integration_repre_list: rep["tags"].append("delete") diff --git a/pype/plugins/maya/create/create_render.py b/pype/plugins/maya/create/create_render.py index 6826d33c580..fa0e2691260 100644 --- a/pype/plugins/maya/create/create_render.py +++ b/pype/plugins/maya/create/create_render.py @@ -188,6 +188,7 @@ def _create_render_settings(self): self.data["tileRendering"] = False self.data["tilesX"] = 2 self.data["tilesY"] = 2 + self.data["convertToScanline"] = False # Disable for now as this feature is not working yet # self.data["assScene"] = False From ab9d8e266ad6b645fee04a696dccd568893f3643 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 8 Sep 2020 17:34:03 +0200 Subject: [PATCH 2/7] fix repre processing and name --- .../global/publish/extract_scanline_exr.py | 17 ++++++++--------- pype/plugins/maya/publish/collect_render.py | 3 ++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index e1ca0e7c1c3..4a34038f7ee 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Convert tiled exrs to scanline if needed.""" +"""Convert exrs in representation to tiled exrs usin oiio tools.""" import os import copy @@ -18,13 +18,14 @@ class ExtractScanlineExr(pyblish.api.InstancePlugin): def process(self, instance): """Plugin entry point.""" - anatomy = instance.context.data['anatomy'] - # get representation and loop them representations = instance.data["representations"] + representations_new = [] for repre in representations: + self.log.info( + "Processnig representation {}".format(repre.get("name"))) tags = repre.get("tags", []) if "toScanline" not in tags: continue @@ -41,7 +42,6 @@ def process(self, instance): self.log.info("We have a single frame") stagingdir = os.path.normpath(repre.get("stagingDir")) - stagingdir = anatomy.fill_root(stagingdir) oiio_tool_path = os.getenv("PYPE_OIIO_PATH", "") @@ -57,7 +57,7 @@ def process(self, instance): oiio_cmd.append("-o") new_file = f"_scanline_{file}" new_files.append(new_file) - oiio_cmd.append(stagingdir, new_file) + oiio_cmd.append(os.path.join(stagingdir, new_file)) subprocess_exr = " ".join(oiio_cmd) self.log.info(f"running: {subprocess_exr}") @@ -74,12 +74,11 @@ def process(self, instance): representation = copy.deepcopy(repre) - representation['name'] = 'scanline exr' + representation['name'] = 'scanline_exr' representation['files'] = new_files if len(new_files) > 1 else new_files[0] # noqa: E501 - representation['tags'].remove("toScanline") + representation['tags'] = [] - # add representation self.log.debug("Adding: {}".format(representation)) representations_new.append(representation) - instance.data["representations"] = representations_new + instance.data["representations"] += representations_new diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py index 91230fcc462..093827809c1 100644 --- a/pype/plugins/maya/publish/collect_render.py +++ b/pype/plugins/maya/publish/collect_render.py @@ -246,7 +246,8 @@ def process(self, context): "tileRendering": render_instance.data.get("tileRendering") or False, # noqa: E501 "tilesX": render_instance.data.get("tilesX") or 2, "tilesY": render_instance.data.get("tilesY") or 2, - "priority": render_instance.data.get("priority") + "priority": render_instance.data.get("priority"), + "convertToScanline": render_instance.data.get("convertToScanline") or False # noqa: E501 } # Apply each user defined attribute as data From dc883fe0a41ef4157340047a84cdbd30e19d4888 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 8 Sep 2020 19:02:45 +0200 Subject: [PATCH 3/7] improve log texts --- pype/plugins/global/publish/extract_scanline_exr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index 4a34038f7ee..2f68a2af0d6 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -25,13 +25,15 @@ def process(self, instance): for repre in representations: self.log.info( - "Processnig representation {}".format(repre.get("name"))) + "Processing representation {}".format(repre.get("name"))) tags = repre.get("tags", []) if "toScanline" not in tags: + self.log.info("- missing toScanline tag") continue # run only on exrs if repre.get("ext") != "exr": + self.log.info("- not EXR files") continue if not isinstance(repre['files'], (list, tuple)): From a38446095a21aa41f497bdf3a64baf04e14afd8f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 9 Sep 2020 10:59:55 +0200 Subject: [PATCH 4/7] add scanline processing to flat sequence too --- pype/plugins/global/publish/submit_publish_job.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 2d6d1efff83..5c46746e026 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -494,6 +494,7 @@ def _create_instances_for_aov(self, instance_data, exp_files): # support conversion from tiled to scanline if instance_data.get("convertToScanline"): + self.log.info("Adding scanline conversion.") rep["tags"].append("toScanline") # poor man exclusion @@ -586,6 +587,11 @@ def _get_representations(self, instance, exp_files): if instance.get("multipartExr", False): rep["tags"].append("multipartExr") + # support conversion from tiled to scanline + if instance.get("convertToScanline"): + self.log.info("Adding scanline conversion.") + rep["tags"].append("toScanline") + representations.append(rep) self._solve_families(instance, preview) From 11cf4056a7aeaa384ab867bf7460e0c61f34222a Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 1 Oct 2020 14:53:03 +0200 Subject: [PATCH 5/7] replace files in current representation --- .../global/publish/extract_scanline_exr.py | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index 2f68a2af0d6..1c3873b0210 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Convert exrs in representation to tiled exrs usin oiio tools.""" import os -import copy +import shutil import pyblish.api import pype.api @@ -47,40 +47,44 @@ def process(self, instance): oiio_tool_path = os.getenv("PYPE_OIIO_PATH", "") - new_files = [] for file in input_files: + original_name = os.path.join(stagingdir, file) + temp_name = os.path.join(stagingdir, "__{}".format(file)) + # move original render to temp location + shutil.move(original_name, temp_name) oiio_cmd = [] oiio_cmd.append(oiio_tool_path) oiio_cmd.append( - os.path.join(stagingdir, file) + os.path.join(stagingdir, temp_name) ) oiio_cmd.append("--scanline") oiio_cmd.append("-o") - new_file = f"_scanline_{file}" - new_files.append(new_file) - oiio_cmd.append(os.path.join(stagingdir, new_file)) + oiio_cmd.append(os.path.join(stagingdir, original_name)) subprocess_exr = " ".join(oiio_cmd) self.log.info(f"running: {subprocess_exr}") pype.api.subprocess(subprocess_exr) # raise error if there is no ouptput - if not os.path.exists(os.path.join(stagingdir, new_file)): + if not os.path.exists(os.path.join(stagingdir, original_name)): self.log.error( - f"File {new_file} was not produced by oiio tool!") + ("File {} was not converted " + "by oiio tool!").format(original_name)) raise AssertionError("OIIO tool conversion failed") - - if "representations" not in instance.data: - instance.data["representations"] = [] - - representation = copy.deepcopy(repre) - - representation['name'] = 'scanline_exr' - representation['files'] = new_files if len(new_files) > 1 else new_files[0] # noqa: E501 - representation['tags'] = [] - - self.log.debug("Adding: {}".format(representation)) - representations_new.append(representation) + else: + try: + shutil.remove(temp_name) + except OSError as e: + self.log.warning("Unable to delete temp file") + self.log.warning(e) + + repre['name'] = 'exr' + repre['outputName'] = "scanline exr" + try: + repre['tags'].remove('toScanline') + except ValueError: + # no `toScanline` tag present + pass instance.data["representations"] += representations_new From 3f628d12672e7f86ea1aa603ff04c8ea9ba48546 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 1 Oct 2020 16:19:37 +0200 Subject: [PATCH 6/7] minor fixes --- pype/plugins/global/publish/extract_scanline_exr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index 1c3873b0210..d829f198b67 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -28,7 +28,7 @@ def process(self, instance): "Processing representation {}".format(repre.get("name"))) tags = repre.get("tags", []) if "toScanline" not in tags: - self.log.info("- missing toScanline tag") + self.log.info(" - missing toScanline tag") continue # run only on exrs @@ -38,10 +38,10 @@ def process(self, instance): if not isinstance(repre['files'], (list, tuple)): input_files = [repre['files']] - self.log.info("We have a sequence.") + self.log.info("We have a single frame") else: input_files = repre['files'] - self.log.info("We have a single frame") + self.log.info("We have a sequence") stagingdir = os.path.normpath(repre.get("stagingDir")) @@ -74,7 +74,7 @@ def process(self, instance): raise AssertionError("OIIO tool conversion failed") else: try: - shutil.remove(temp_name) + os.remove(temp_name) except OSError as e: self.log.warning("Unable to delete temp file") self.log.warning(e) From 5368632c859466b18ec879ab3c393d557aafc0a3 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 6 Oct 2020 17:45:38 +0200 Subject: [PATCH 7/7] removed repre output name --- pype/plugins/global/publish/extract_scanline_exr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index d829f198b67..ca62476ab2d 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -80,7 +80,6 @@ def process(self, instance): self.log.warning(e) repre['name'] = 'exr' - repre['outputName'] = "scanline exr" try: repre['tags'].remove('toScanline') except ValueError: