Houdini: Add support to split Deadline render tasks in export + render

openpype/hosts/houdini/api/lib.py

@@ -121,8 +121,64 @@ def get_id_required_nodes():
  return list(nodes)
 
 
+def get_export_parameter(node):
+ """Return the export output parameter of the given node
+
+ Example:
+ root = hou.node("/obj")
+ my_alembic_node = root.createNode("alembic")
+ get_output_parameter(my_alembic_node)
+ # Result: "output"
+
+ Args:
+ node(hou.Node): node instance
+
+ Returns:
+ hou.Parm
+
+ """
+ node_type = node.type().description()
+
+ # Ensures the proper Take is selected for each ROP to retrieve the correct
+ # ifd
+ try:
+ rop_take = hou.takes.findTake(node.parm("take").eval())
+ if rop_take is not None:
+ hou.takes.setCurrentTake(rop_take)
+ except AttributeError:
+ # hou object doesn't always have the 'takes' attribute
+ pass
+
+ if node_type == "Mantra" and node.parm("soho_outputmode").eval():
+ return node.parm("soho_diskfile")
+ elif node_type == "Alfred":
+ return node.parm("alf_diskfile")
+ elif (node_type == "RenderMan" or node_type == "RenderMan RIS"):
+ pre_ris22 = node.parm("rib_outputmode") and \
+ node.parm("rib_outputmode").eval()
+ ris22 = node.parm("diskfile") and node.parm("diskfile").eval()
+ if pre_ris22 or ris22:
+ return node.parm("soho_diskfile")
+ elif node_type == "Redshift" and node.parm("RS_archive_enable").eval():
+ return node.parm("RS_archive_file")
+ elif node_type == "Wedge" and node.parm("driver").eval():
+ return get_export_parameter(node.node(node.parm("driver").eval()))
+ elif node_type == "Arnold":
+ return node.parm("ar_ass_file")
+ elif node_type == "Alembic" and node.parm("use_sop_path").eval():
+ return node.parm("sop_path")
+ elif node_type == "Shotgun Mantra" and node.parm("soho_outputmode").eval():
+ return node.parm("sgtk_soho_diskfile")
+ elif node_type == "Shotgun Alembic" and node.parm("use_sop_path").eval():
+ return node.parm("sop_path")
+ elif node.type().nameWithCategory() == "Driver/vray_renderer":
+ return node.parm("render_export_filepath")
+
+ raise TypeError("Node type '%s' not supported" % node_type)
+
+
 def get_output_parameter(node):
- """Return the render output parameter name of the given node
+ """Return the render output parameter of the given node
 
  Example:
  root = hou.node("/obj")
@@ -137,13 +193,14 @@ def get_output_parameter(node):
  hou.Parm
 
  """
+ node_type = node.type().description()
+ category = node.type().category().name()
 
- node_type = node.type().name()
- if node_type == "geometry":
+ # Figure out which type of node is being rendered
+ if node_type == "Geometry" or node_type == "Filmbox FBX" or \
+ (node_type == "ROP Output Driver" and category == "Sop"):
  return node.parm("sopoutput")
- elif node_type == "alembic":
- return node.parm("filename")
- elif node_type == "comp":
+ elif node_type == "Composite":
  return node.parm("copoutput")
  elif node_type == "opengl":
  return node.parm("picture")
@@ -155,6 +212,15 @@ def get_output_parameter(node):
  elif node_type == "ifd":
  if node.evalParm("soho_outputmode"):
  return node.parm("soho_diskfile")
+ elif node_type == "Octane":
+ return node.parm("HO_img_fileName")
+ elif node_type == "Fetch":
+ inner_node = node.node(node.parm("source").eval())
+ if inner_node:
+ return get_output_parameter(inner_node)
+ elif node.type().nameWithCategory() == "Driver/vray_renderer":
+ return node.parm("SettingsOutput_img_file_path")
+
  raise TypeError("Node type '%s' not supported" % node_type)
 
 

openpype/hosts/houdini/plugins/create/create_arnold_rop.py

@@ -13,6 +13,9 @@ class CreateArnoldRop(plugin.HoudiniCreator):
  # Default extension
  ext = "exr"
 
+ # Default to split export and render jobs
+ export_job = True
+
  def create(self, subset_name, instance_data, pre_create_data):
  import hou
 
@@ -48,6 +51,15 @@ def create(self, subset_name, instance_data, pre_create_data):
  "ar_exr_half_precision": 1 # half precision
  }
 
+ if pre_create_data.get("export_job"):
+ ass_filepath = \
+ "{export_dir}{subset_name}/{subset_name}.$F4.ass".format(
+ export_dir=hou.text.expandString("$HIP/pyblish/ass/"),
+ subset_name=subset_name,
+ )
+ parms["ar_ass_export_enable"] = 1
+ parms["ar_ass_file"] = ass_filepath
+
  instance_node.setParms(parms)
 
  # Lock any parameters in this list
@@ -66,6 +78,9 @@ def get_pre_create_attr_defs(self):
  BoolDef("farm",
  label="Submitting to Farm",
  default=True),
+ BoolDef("export_job",
+ label="Split export and render jobs",
+ default=self.export_job),
  EnumDef("image_format",
  image_format_enum,
  default=self.ext,

openpype/hosts/houdini/plugins/create/create_mantra_rop.py

@@ -12,6 +12,9 @@ class CreateMantraROP(plugin.HoudiniCreator):
  family = "mantra_rop"
  icon = "magic"
 
+ # Default to split export and render jobs
+ export_job = True
+
  def create(self, subset_name, instance_data, pre_create_data):
  import hou # noqa
 
@@ -44,6 +47,15 @@ def create(self, subset_name, instance_data, pre_create_data):
  "vm_picture": filepath,
  }
 
+ if pre_create_data.get("export_job"):
+ ifd_filepath = \
+ "{export_dir}{subset_name}/{subset_name}.$F4.ifd".format(
+ export_dir=hou.text.expandString("$HIP/pyblish/ifd/"),
+ subset_name=subset_name,
+ )
+ parms["soho_outputmode"] = 1
+ parms["soho_diskfile"] = ifd_filepath
+
  if self.selected_nodes:
  # If camera found in selection
  # we will use as render camera
@@ -78,6 +90,9 @@ def get_pre_create_attr_defs(self):
  BoolDef("farm",
  label="Submitting to Farm",
  default=True),
+ BoolDef("export_job",
+ label="Split export and render jobs",
+ default=self.export_job),
  EnumDef("image_format",
  image_format_enum,
  default="exr",

openpype/hosts/houdini/plugins/create/create_vray_rop.py

@@ -16,6 +16,9 @@ class CreateVrayROP(plugin.HoudiniCreator):
  icon = "magic"
  ext = "exr"
 
+ # Default to split export and render jobs
+ export_job = True
+
  def create(self, subset_name, instance_data, pre_create_data):
 
  instance_data.pop("active", None)
@@ -52,6 +55,17 @@ def create(self, subset_name, instance_data, pre_create_data):
  "SettingsEXR_bits_per_channel": "16" # half precision
  }
 
+ if pre_create_data.get("export_job"):
+ scene_filepath = \
+ "{export_dir}{subset_name}/{subset_name}.$F4.vrscene".format(
+ export_dir=hou.text.expandString("$HIP/pyblish/vrscene/"),
+ subset_name=subset_name,
+ )
+ # Setting render_export_mode to "2" because that's for
+ # "Export only" ("1" is for "Export & Render")
+ parms["render_export_mode"] = "2"
+ parms["render_export_filepath"] = scene_filepath
+
  if self.selected_nodes:
  # set up the render camera from the selected node
  camera = None
@@ -140,6 +154,9 @@ def get_pre_create_attr_defs(self):
  BoolDef("farm",
  label="Submitting to Farm",
  default=True),
+ BoolDef("export_job",
+ label="Split export and render jobs",
+ default=self.export_job),
  EnumDef("image_format",
  image_format_enum,
  default=self.ext,

openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py

@@ -40,6 +40,25 @@ def process(self, instance):
  default_prefix = evalParmNoFrame(rop, "ar_picture")
  render_products = []
 
+ # Store whether we are splitting the render job (export + render)
+ export_job = bool(rop.parm("ar_ass_export_enable").eval())
+ instance.data["exportJob"] = export_job
+ export_prefix = None
+ export_products = []
+ if export_job:
+ export_prefix = evalParmNoFrame(
+ rop, "ar_ass_file", pad_character="0"
+ )
+ beauty_export_product = self.get_render_product_name(
+ prefix=export_prefix,
+ suffix=None)
+ export_products.append(beauty_export_product)
+ self.log.debug(
+ "Found export product: {}".format(beauty_export_product)
+ )
+ instance.data["ifdFile"] = beauty_export_product
+ instance.data["exportFiles"] = list(export_products)
+
  # Default beauty AOV
  beauty_product = self.get_render_product_name(prefix=default_prefix,
  suffix=None)

openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py

@@ -44,6 +44,25 @@ def process(self, instance):
  default_prefix = evalParmNoFrame(rop, "vm_picture")
  render_products = []
 
+ # Store whether we are splitting the render job (export + render)
+ export_job = bool(rop.parm("soho_outputmode").eval())
+ instance.data["exportJob"] = export_job
+ export_prefix = None
+ export_products = []
+ if export_job:
+ export_prefix = evalParmNoFrame(
+ rop, "soho_diskfile", pad_character="0"
+ )
+ beauty_export_product = self.get_render_product_name(
+ prefix=export_prefix,
+ suffix=None)
+ export_products.append(beauty_export_product)
+ self.log.debug(
+ "Found export product: {}".format(beauty_export_product)
+ )
+ instance.data["ifdFile"] = beauty_export_product
+ instance.data["exportFiles"] = list(export_products)
+
  # Default beauty AOV
  beauty_product = self.get_render_product_name(
  prefix=default_prefix, suffix=None

openpype/hosts/houdini/plugins/publish/collect_vray_rop.py

@@ -45,7 +45,26 @@ def process(self, instance):
  render_products = []
  # TODO: add render elements if render element
 
- beauty_product = self.get_beauty_render_product(default_prefix)
+ # Store whether we are splitting the render job in an export + render
+ export_job = rop.parm("render_export_mode").eval() == "2"
+ instance.data["exportJob"] = export_job
+ export_prefix = None
+ export_products = []
+ if export_job:
+ export_prefix = evalParmNoFrame(
+ rop, "render_export_filepath", pad_character="0"
+ )
+ beauty_export_product = self.get_render_product_name(
+ prefix=export_prefix,
+ suffix=None)
+ export_products.append(beauty_export_product)
+ self.log.debug(
+ "Found export product: {}".format(beauty_export_product)
+ )
+ instance.data["ifdFile"] = beauty_export_product
+ instance.data["exportFiles"] = list(export_products)
+
+ beauty_product = self.get_render_product_name(default_prefix)
  render_products.append(beauty_product)
  files_by_aov = {
  "RGB Color": self.generate_expected_files(instance,
@@ -79,7 +98,7 @@ def process(self, instance):
  instance.data["colorspaceDisplay"] = colorspace_data["display"]
  instance.data["colorspaceView"] = colorspace_data["view"]
 
- def get_beauty_render_product(self, prefix, suffix="<reName>"):
+ def get_render_product_name(self, prefix, suffix="<reName>"):
  """Return the beauty output filename if render element enabled
  """
  # Remove aov suffix from the product: `prefix.aov_suffix` -> `prefix`

openpype/modules/deadline/abstract_submit_deadline.py

@@ -460,7 +460,22 @@ def process(self, instance):
  self.plugin_info = self.get_plugin_info()
  self.aux_files = self.get_aux_files()
 
- self.process_submission()
+ job_id = self.process_submission()
+ self.log.info("Submitted job to Deadline: {}.".format(job_id))
+
+ # TODO: Find a way that's more generic and not render type specific
+ export_job = instance.data["exportJob"]
+ if export_job:
+ self.log.info("Splitting export and render in two jobs")
+ self.log.info("Export job id: %s", job_id)
+ render_job_info = self.get_job_info(dependency_job_ids=[job_id])
+ render_plugin_info = self.get_plugin_info(job_type="render")
+ payload = self.assemble_payload(
+ job_info=render_job_info,
+ plugin_info=render_plugin_info
+ )
+ render_job_id = self.submit(payload)
+ self.log.info("Render job id: %s", render_job_id)
 
  def process_submission(self):
  """Process data for submission.