Houdini: Farm caching submission to Deadline

openpype/hosts/houdini/api/lib.py

@@ -138,7 +138,9 @@ def get_output_parameter(node):
  return node.parm("ar_ass_file")
  elif node_type == "Redshift_Proxy_Output":
  return node.parm("RS_archive_file")
-
+ elif node_type == "ifd":
+ if node.evalParm("soho_outputmode"):
+ return node.parm("soho_diskfile")
  raise TypeError("Node type '%s' not supported" % node_type)
 
 

openpype/hosts/houdini/plugins/create/create_arnold_ass.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """Creator plugin for creating Arnold ASS files."""
 from openpype.hosts.houdini.api import plugin
+from openpype.lib import BoolDef
 
 
 class CreateArnoldAss(plugin.HoudiniCreator):
@@ -21,6 +22,9 @@ def create(self, subset_name, instance_data, pre_create_data):
 
  instance_data.pop("active", None)
  instance_data.update({"node_type": "arnold"})
+ creator_attributes = instance_data.setdefault(
+ "creator_attributes", dict())
+ creator_attributes["farm"] = pre_create_data["farm"]
 
  instance = super(CreateArnoldAss, self).create(
  subset_name,
@@ -52,3 +56,15 @@ def create(self, subset_name, instance_data, pre_create_data):
  # Lock any parameters in this list
  to_lock = ["ar_ass_export_enable", "family", "id"]
  self.lock_parameters(instance_node, to_lock)
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef("farm",
+ label="Submitting to Farm",
+ default=False)
+ ]
+
+ def get_pre_create_attr_defs(self):
+ attrs = super().get_pre_create_attr_defs()
+ # Use same attributes as for instance attributes
+ return attrs + self.get_instance_attr_defs()

openpype/hosts/houdini/plugins/create/create_bgeo.py

@@ -2,7 +2,7 @@
 """Creator plugin for creating pointcache bgeo files."""
 from openpype.hosts.houdini.api import plugin
 from openpype.pipeline import CreatedInstance, CreatorError
-from openpype.lib import EnumDef
+from openpype.lib import EnumDef, BoolDef
 
 
 class CreateBGEO(plugin.HoudiniCreator):
@@ -18,6 +18,9 @@ def create(self, subset_name, instance_data, pre_create_data):
  instance_data.pop("active", None)
 
  instance_data.update({"node_type": "geometry"})
+ creator_attributes = instance_data.setdefault(
+ "creator_attributes", dict())
+ creator_attributes["farm"] = pre_create_data["farm"]
 
  instance = super(CreateBGEO, self).create(
  subset_name,
@@ -58,6 +61,13 @@ def create(self, subset_name, instance_data, pre_create_data):
 
  instance_node.setParms(parms)
 
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef("farm",
+ label="Submitting to Farm",
+ default=False)
+ ]
+
  def get_pre_create_attr_defs(self):
  attrs = super().get_pre_create_attr_defs()
  bgeo_enum = [
@@ -88,5 +98,5 @@ def get_pre_create_attr_defs(self):
  ]
 
  return attrs + [
- EnumDef("bgeo_type", bgeo_enum, label="BGEO Options"),
- ]
+ EnumDef("bgeo_type", bgeo_enum, label="BGEO Options")
+ ] + self.get_instance_attr_defs()

openpype/hosts/houdini/plugins/create/create_mantra_ifd.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""Creator plugin for creating pointcache alembics."""
+from openpype.hosts.houdini.api import plugin
+from openpype.pipeline import CreatedInstance
+from openpype.lib import BoolDef
+
+
+class CreateMantraIFD(plugin.HoudiniCreator):
+ """Mantra .ifd Archive"""
+ identifier = "io.openpype.creators.houdini.mantraifd"
+ label = "Mantra IFD"
+ family = "mantraifd"
+ icon = "gears"
+
+ def create(self, subset_name, instance_data, pre_create_data):
+ import hou
+ instance_data.pop("active", None)
+ instance_data.update({"node_type": "ifd"})
+ creator_attributes = instance_data.setdefault(
+ "creator_attributes", dict())
+ creator_attributes["farm"] = pre_create_data["farm"]
+ instance = super(CreateMantraIFD, self).create(
+ subset_name,
+ instance_data,
+ pre_create_data) # type: CreatedInstance
+
+ instance_node = hou.node(instance.get("instance_node"))
+
+ filepath = "{}{}".format(
+ hou.text.expandString("$HIP/pyblish/"),
+ "{}.$F4.ifd".format(subset_name))
+ parms = {
+ # Render frame range
+ "trange": 1,
+ # Arnold ROP settings
+ "soho_diskfile": filepath,
+ "soho_outputmode": 1
+ }
+
+ instance_node.setParms(parms)
+
+ # Lock any parameters in this list
+ to_lock = ["soho_outputmode", "family", "id"]
+ self.lock_parameters(instance_node, to_lock)
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef("farm",
+ label="Submitting to Farm",
+ default=False)
+ ]
+
+ def get_pre_create_attr_defs(self):
+ attrs = super().get_pre_create_attr_defs()
+ # Use same attributes as for instance attributes
+ return attrs + self.get_instance_attr_defs()

openpype/hosts/houdini/plugins/create/create_pointcache.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 """Creator plugin for creating pointcache alembics."""
 from openpype.hosts.houdini.api import plugin
+from openpype.lib import BoolDef
 
 import hou
 
 
+
 class CreatePointCache(plugin.HoudiniCreator):
  """Alembic ROP to pointcache"""
  identifier = "io.openpype.creators.houdini.pointcache"
@@ -15,6 +17,9 @@ class CreatePointCache(plugin.HoudiniCreator):
  def create(self, subset_name, instance_data, pre_create_data):
  instance_data.pop("active", None)
  instance_data.update({"node_type": "alembic"})
+ creator_attributes = instance_data.setdefault(
+ "creator_attributes", dict())
+ creator_attributes["farm"] = pre_create_data["farm"]
 
  instance = super(CreatePointCache, self).create(
  subset_name,
@@ -105,3 +110,15 @@ def get_obj_output(self, obj_node):
  else:
  return min(outputs,
  key=lambda node: node.evalParm('outputidx'))
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef("farm",
+ label="Submitting to Farm",
+ default=False)
+ ]
+
+ def get_pre_create_attr_defs(self):
+ attrs = super().get_pre_create_attr_defs()
+ # Use same attributes as for instance attributes
+ return attrs + self.get_instance_attr_defs()

openpype/hosts/houdini/plugins/create/create_redshift_proxy.py

@@ -2,6 +2,7 @@
 """Creator plugin for creating Redshift proxies."""
 from openpype.hosts.houdini.api import plugin
 from openpype.pipeline import CreatedInstance
+from openpype.lib import BoolDef
 
 
 class CreateRedshiftProxy(plugin.HoudiniCreator):
@@ -24,6 +25,9 @@ def create(self, subset_name, instance_data, pre_create_data):
  # TODO: Somehow enforce so that it only shows the original limited
  # attributes of the Redshift_Proxy_Output node type
  instance_data.update({"node_type": "Redshift_Proxy_Output"})
+ creator_attributes = instance_data.setdefault(
+ "creator_attributes", dict())
+ creator_attributes["farm"] = pre_create_data["farm"]
 
  instance = super(CreateRedshiftProxy, self).create(
  subset_name,
@@ -44,3 +48,15 @@ def create(self, subset_name, instance_data, pre_create_data):
  # Lock some Avalon attributes
  to_lock = ["family", "id", "prim_to_detail_pattern"]
  self.lock_parameters(instance_node, to_lock)
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef("farm",
+ label="Submitting to Farm",
+ default=False)
+ ]
+
+ def get_pre_create_attr_defs(self):
+ attrs = super().get_pre_create_attr_defs()
+ # Use same attributes as for instance attributes
+ return attrs + self.get_instance_attr_defs()

openpype/hosts/houdini/plugins/create/create_vbd_cache.py

@@ -2,6 +2,7 @@
 """Creator plugin for creating VDB Caches."""
 from openpype.hosts.houdini.api import plugin
 from openpype.pipeline import CreatedInstance
+from openpype.lib import BoolDef
 
 import hou
 
@@ -19,15 +20,20 @@ def create(self, subset_name, instance_data, pre_create_data):
 
  instance_data.pop("active", None)
  instance_data.update({"node_type": "geometry"})
-
+ creator_attributes = instance_data.setdefault(
+ "creator_attributes", dict())
+ creator_attributes["farm"] = pre_create_data["farm"]
  instance = super(CreateVDBCache, self).create(
  subset_name,
  instance_data,
  pre_create_data) # type: CreatedInstance
 
  instance_node = hou.node(instance.get("instance_node"))
+ file_path = "{}{}".format(
+ hou.text.expandString("$HIP/pyblish/"),
+ "{}.$F4.vdb".format(subset_name))
  parms = {
- "sopoutput": "$HIP/pyblish/{}.$F4.vdb".format(subset_name),
+ "sopoutput": file_path,
  "initsim": True,
  "trange": 1
  }
@@ -102,3 +108,15 @@ def get_obj_output(self, obj_node):
  else:
  return min(outputs,
  key=lambda node: node.evalParm('outputidx'))
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef("farm",
+ label="Submitting to Farm",
+ default=False)
+ ]
+
+ def get_pre_create_attr_defs(self):
+ attrs = super().get_pre_create_attr_defs()
+ # Use same attributes as for instance attributes
+ return attrs + self.get_instance_attr_defs()

openpype/hosts/houdini/plugins/publish/collect_cache_farm.py

@@ -0,0 +1,75 @@
+import os
+import pyblish.api
+import hou
+from openpype.hosts.houdini.api import lib
+
+
+class CollectDataforCache(pyblish.api.InstancePlugin):
+ """Collect data for caching to Deadline."""
+
+ order = pyblish.api.CollectorOrder + 0.04
+ families = ["ass", "pointcache",
+ "mantraifd", "redshiftproxy",
+ "vdbcache"]
+ hosts = ["houdini"]
+ targets = ["local", "remote"]
+ label = "Collect Data for Cache"
+
+ def process(self, instance):
+ creator_attribute = instance.data["creator_attributes"]
+ farm_enabled = creator_attribute["farm"]
+ instance.data["farm"] = farm_enabled
+ if not farm_enabled:
+ self.log.debug("Caching on farm is disabled. "
+ "Skipping farm collecting.")
+ return
+ # Why do we need this particular collector to collect the expected
+ # output files from a ROP node. Don't we have a dedicated collector
+ # for that yet?
+ # Collect expected files
+ ropnode = hou.node(instance.data["instance_node"])
+ output_parm = lib.get_output_parameter(ropnode)
+ expected_filepath = output_parm.eval()
+ instance.data.setdefault("files", list())
+ instance.data.setdefault("expectedFiles", list())
+ if instance.data.get("frames"):
+ files = self.get_files(instance, expected_filepath)
+ # list of files
+ instance.data["files"].extend(files)
+ else:
+ # single file
+ instance.data["files"].append(output_parm.eval())
+ cache_files = {"_": instance.data["files"]}
+ # Convert instance family to pointcache if it is bgeo or abc
+ # because ???
+ for family in instance.data["families"]:
+ if family == "bgeo" or "abc":
+ instance.data["family"] = "pointcache"
+ break
+ instance.data.update({
+ "plugin": "Houdini",
+ "publish": True
+ })
+ instance.data["families"].append("publish.hou")
+ instance.data["expectedFiles"].append(cache_files)
+
+ self.log.debug("{}".format(instance.data))
+
+ def get_files(self, instance, output_parm):
+ """Get the files with the frame range data
+
+ Args:
+ instance (_type_): instance
+ output_parm (_type_): path of output parameter
+
+ Returns:
+ files: a list of files
+ """
+ directory = os.path.dirname(output_parm)
+
+ files = [
+ os.path.join(directory, frame).replace("\\", "/")
+ for frame in instance.data["frames"]
+ ]
+
+ return files

openpype/hosts/houdini/plugins/publish/collect_chunk_size.py

@@ -0,0 +1,39 @@
+import pyblish.api
+from openpype.lib import NumberDef
+from openpype.pipeline import OpenPypePyblishPluginMixin
+
+
+class CollectChunkSize(pyblish.api.InstancePlugin,
+ OpenPypePyblishPluginMixin):
+ """Collect chunk size for cache submission to Deadline."""
+
+ order = pyblish.api.CollectorOrder + 0.05
+ families = ["ass", "pointcache",
+ "vdbcache", "mantraifd",
+ "redshiftproxy"]
+ hosts = ["houdini"]
+ targets = ["local", "remote"]
+ label = "Collect Chunk Size"
+ chunkSize = 999999
+
+ def process(self, instance):
+ # need to get the chunk size info from the setting
+ attr_values = self.get_attr_values_from_data(instance.data)
+ instance.data["chunkSize"] = attr_values.get("chunkSize")
+
+ @classmethod
+ def apply_settings(cls, project_settings):
+ project_setting = project_settings["houdini"]["publish"]["CollectChunkSize"] # noqa
+ cls.chunkSize = project_setting["chunk_size"]
+
+ @classmethod
+ def get_attribute_defs(cls):
+ return [
+ NumberDef("chunkSize",
+ minimum=1,
+ maximum=999999,
+ decimals=0,
+ default=cls.chunkSize,
+ label="Frame Per Task")
+
+ ]

openpype/hosts/houdini/plugins/publish/collect_frames.py

@@ -14,7 +14,8 @@ class CollectFrames(pyblish.api.InstancePlugin):
  order = pyblish.api.CollectorOrder + 0.01
  label = "Collect Frames"
  families = ["vdbcache", "imagesequence", "ass",
- "redshiftproxy", "review", "bgeo"]
+ "mantraifd", "redshiftproxy", "review",
+ "bgeo"]
 
  def process(self, instance):