Merge pull request #1923 from pypeclub/enhancement/maya-deadline-scen…

…e-patching

Maya: Scene patching 🩹on submission to Deadline

openpype/modules/deadline/plugins/publish/submit_maya_deadline.py

@@ -199,7 +199,7 @@ def get_renderer_variables(renderlayer, root):
  if extension is None:
  extension = "png"
 
- if extension == "exr (multichannel)" or extension == "exr (deep)":
+ if extension in ["exr (multichannel)", "exr (deep)"]:
  extension = "exr"
 
  prefix_attr = "vraySettings.fileNamePrefix"
@@ -295,57 +295,70 @@ def process(self, instance):
  instance.data["toBeRenderedOn"] = "deadline"
 
  filepath = None
+ patches = (
+ context.data["project_settings"].get(
+ "deadline", {}).get(
+ "publish", {}).get(
+ "MayaSubmitDeadline", {}).get(
+ "scene_patches", {})
+ )
 
  # Handle render/export from published scene or not ------------------
  if self.use_published:
+ patched_files = []
  for i in context:
- if "workfile" in i.data["families"]:
- assert i.data["publish"] is True, (
- "Workfile (scene) must be published along")
- template_data = i.data.get("anatomyData")
- rep = i.data.get("representations")[0].get("name")
- template_data["representation"] = rep
- template_data["ext"] = rep
- template_data["comment"] = None
- anatomy_filled = anatomy.format(template_data)
- template_filled = anatomy_filled["publish"]["path"]
- filepath = os.path.normpath(template_filled)
- self.log.info("Using published scene for render {}".format(
- filepath))
-
- if not os.path.exists(filepath):
- self.log.error("published scene does not exist!")
- raise
- # now we need to switch scene in expected files
- # because <scene> token will now point to published
- # scene file and that might differ from current one
- new_scene = os.path.splitext(
- os.path.basename(filepath))[0]
- orig_scene = os.path.splitext(
- os.path.basename(context.data["currentFile"]))[0]
- exp = instance.data.get("expectedFiles")
-
- if isinstance(exp[0], dict):
- # we have aovs and we need to iterate over them
- new_exp = {}
- for aov, files in exp[0].items():
- replaced_files = []
- for f in files:
- replaced_files.append(
- f.replace(orig_scene, new_scene)
- )
- new_exp[aov] = replaced_files
- instance.data["expectedFiles"] = [new_exp]
- else:
- new_exp = []
- for f in exp:
- new_exp.append(
+ if "workfile" not in i.data["families"]:
+ continue
+ assert i.data["publish"] is True, (
+ "Workfile (scene) must be published along")
+ template_data = i.data.get("anatomyData")
+ rep = i.data.get("representations")[0].get("name")
+ template_data["representation"] = rep
+ template_data["ext"] = rep
+ template_data["comment"] = None
+ anatomy_filled = anatomy.format(template_data)
+ template_filled = anatomy_filled["publish"]["path"]
+ filepath = os.path.normpath(template_filled)
+ self.log.info("Using published scene for render {}".format(
+ filepath))
+
+ if not os.path.exists(filepath):
+ self.log.error("published scene does not exist!")
+ raise
+ # now we need to switch scene in expected files
+ # because <scene> token will now point to published
+ # scene file and that might differ from current one
+ new_scene = os.path.splitext(
+ os.path.basename(filepath))[0]
+ orig_scene = os.path.splitext(
+ os.path.basename(context.data["currentFile"]))[0]
+ exp = instance.data.get("expectedFiles")
+
+ if isinstance(exp[0], dict):
+ # we have aovs and we need to iterate over them
+ new_exp = {}
+ for aov, files in exp[0].items():
+ replaced_files = []
+ for f in files:
+ replaced_files.append(
  f.replace(orig_scene, new_scene)
  )
- instance.data["expectedFiles"] = [new_exp]
- self.log.info("Scene name was switched {} -> {}".format(
- orig_scene, new_scene
- ))
+ new_exp[aov] = replaced_files
+ instance.data["expectedFiles"] = [new_exp]
+ else:
+ new_exp = []
+ for f in exp:
+ new_exp.append(
+ f.replace(orig_scene, new_scene)
+ )
+ instance.data["expectedFiles"] = [new_exp]
+ self.log.info("Scene name was switched {} -> {}".format(
+ orig_scene, new_scene
+ ))
+ # patch workfile is needed
+ if filepath not in patched_files:
+ patched_file = self._patch_workfile(filepath, patches)
+ patched_files.append(patched_file)
 
  all_instances = []
  for result in context.data["results"]:
@@ -868,10 +881,11 @@ def _get_arnold_export_payload(self, data):
  payload["JobInfo"].update(job_info_ext)
  payload["PluginInfo"].update(plugin_info_ext)
 
- envs = []
- for k, v in payload["JobInfo"].items():
- if k.startswith("EnvironmentKeyValue"):
- envs.append(v)
+ envs = [
+ v
+ for k, v in payload["JobInfo"].items()
+ if k.startswith("EnvironmentKeyValue")
+ ]
 
  # add app name to environment
  envs.append(
@@ -892,11 +906,8 @@ def _get_arnold_export_payload(self, data):
  envs.append(
  "OPENPYPE_ASS_EXPORT_STEP={}".format(1))
 
- i = 0
- for e in envs:
+ for i, e in enumerate(envs):
  payload["JobInfo"]["EnvironmentKeyValue{}".format(i)] = e
- i += 1
-
  return payload
 
  def _get_vray_render_payload(self, data):
@@ -1003,7 +1014,7 @@ def _requests_post(self, *args, **kwargs):
 
  """
  if 'verify' not in kwargs:
- kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa
+ kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
  # add 10sec timeout before bailing out
  kwargs['timeout'] = 10
  return requests.post(*args, **kwargs)
@@ -1022,7 +1033,7 @@ def _requests_get(self, *args, **kwargs):
 
  """
  if 'verify' not in kwargs:
- kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa
+ kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
  # add 10sec timeout before bailing out
  kwargs['timeout'] = 10
  return requests.get(*args, **kwargs)
@@ -1069,3 +1080,43 @@ def smart_replace(string, key_values):
  result = filename_zero.replace("\\", "/")
 
  return result
+
+ def _patch_workfile(self, file, patches):
+ # type: (str, dict) -> [str, None]
+ """Patch Maya scene.
+
+ This will take list of patches (lines to add) and apply them to
+ *published* Maya scene file (that is used later for rendering).
+
+ Patches are dict with following structure::
+ {
+ "name": "Name of patch",
+ "regex": "regex of line before patch",
+ "line": "line to insert"
+ }
+
+ Args:
+ file (str): File to patch.
+ patches (dict): Dictionary defining patches.
+
+ Returns:
+ str: Patched file path or None
+
+ """
+ if os.path.splitext(file)[1].lower() != ".ma" or not patches:
+ return None
+
+ compiled_regex = [re.compile(p["regex"]) for p in patches]
+ with open(file, "r+") as pf:
+ scene_data = pf.readlines()
+ for ln, line in enumerate(scene_data):
+ for i, r in enumerate(compiled_regex):
+ if re.match(r, line):
+ scene_data.insert(ln + 1, patches[i]["line"])
+ pf.seek(0)
+ pf.writelines(scene_data)
+ pf.truncate()
+ self.log.info(
+ "Applied {} patch to scene.".format(
+ patches[i]["name"]))
+ return file

openpype/settings/defaults/project_settings/deadline.json

@@ -45,7 +45,8 @@
  "group": "none",
  "limit": [],
  "jobInfo": {},
- "pluginInfo": {}
+ "pluginInfo": {},
+ "scene_patches": []
  },
  "NukeSubmitDeadline": {
  "enabled": true,

openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json

@@ -151,7 +151,7 @@
  "type": "dict",
  "collapsible": true,
  "key": "MayaSubmitDeadline",
- "label": "Submit maya job to deadline",
+ "label": "Submit Maya job to Deadline",
  "checkbox_key": "enabled",
  "children": [
  {
@@ -213,6 +213,31 @@
  "type": "raw-json",
  "key": "pluginInfo",
  "label": "Additional PluginInfo data"
+ },
+ {
+ "type": "list",
+ "key": "scene_patches",
+ "label": "Scene patches",
+ "required_keys": ["name", "regex", "line"],
+ "object_type": {
+ "type": "dict",
+ "children": [
+ {
+ "key": "name",
+ "label": "Patch name",
+ "type": "text"
+ }, {
+ "key": "regex",
+ "label": "Patch regex",
+ "type": "text"
+ }, {
+ "key": "line",
+ "label": "Patch line",
+ "type": "text"
+ }
+ ]
+
+ }
  }
  ]
  },

website/docs/admin_hosts_maya.md

@@ -87,6 +87,26 @@ When you publish your model with top group named like `foo_GRP` it will fail. Bu
 All regexes used here are in Python variant.
 :::
 
+### Maya > Deadline submitter
+This plugin provides connection between Maya and Deadline. It is using [Deadline Webservice](https://docs.thinkboxsoftware.com/products/deadline/10.0/1_User%20Manual/manual/web-service.html) to submit jobs to farm.
+![Maya > Deadline Settings](assets/maya-admin_submit_maya_job_to_deadline.png)
+
+You can set various aspects of scene submission to farm with per-project settings in **Setting UI**.
+
+ - **Optional** will mark sumission plugin optional
+ - **Active** will enable/disable plugin
+ - **Tile Assembler Plugin** will set what should be used to assemble tiles on Deadline. Either **Open Image IO** will be used 
+or Deadlines **Draft Tile Assembler**.
+ - **Use Published scene** enable to render from published scene instead of scene in work area. Rendering from published files is much safer.
+ - **Use Asset dependencies** will mark job pending on farm until asset dependencies are fulfilled - for example Deadline will wait for scene file to be synced to cloud, etc.
+ - **Group name** use specific Deadline group for the job.
+ - **Limit Groups** use these Deadline Limit groups for the job.
+ - **Additional `JobInfo` data** JSON of additional Deadline options that will be embedded in `JobInfo` part of the submission data.
+ - **Additional `PluginInfo` data** JSON of additional Deadline options that will be embedded in `PluginInfo` part of the submission data.
+ - **Scene patches** - configure mechanism to add additional lines to published Maya Ascii scene files before they are used for rendering.
+This is useful to fix some specific renderer glitches and advanced hacking of Maya Scene files. `Patch name` is label for patch for easier orientation.
+`Patch regex` is regex used to find line in file, after `Patch line` string is inserted. Note that you need to add line ending.
+
 ## Custom Menu
 You can add your custom tools menu into Maya by extending definitions in **Maya -> Scripts Menu Definition**.
 ![Custom menu definition](assets/maya-admin_scriptsmenu.png)