diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index a652da77865..6b52e4b387f 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -199,7 +199,7 @@ def get_renderer_variables(renderlayer, root): if extension is None: extension = "png" - if extension == "exr (multichannel)" or extension == "exr (deep)": + if extension in ["exr (multichannel)", "exr (deep)"]: extension = "exr" prefix_attr = "vraySettings.fileNamePrefix" @@ -295,57 +295,70 @@ def process(self, instance): instance.data["toBeRenderedOn"] = "deadline" filepath = None + patches = ( + context.data["project_settings"].get( + "deadline", {}).get( + "publish", {}).get( + "MayaSubmitDeadline", {}).get( + "scene_patches", {}) + ) # Handle render/export from published scene or not ------------------ if self.use_published: + patched_files = [] for i in context: - if "workfile" in i.data["families"]: - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0].get("name") - template_data["representation"] = rep - template_data["ext"] = rep - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - filepath = os.path.normpath(template_filled) - self.log.info("Using published scene for render {}".format( - filepath)) - - if not os.path.exists(filepath): - self.log.error("published scene does not exist!") - raise - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext( - os.path.basename(filepath))[0] - orig_scene = os.path.splitext( - os.path.basename(context.data["currentFile"]))[0] - exp = instance.data.get("expectedFiles") - - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - f.replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( + if "workfile" not in i.data["families"]: + continue + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") + template_data = i.data.get("anatomyData") + rep = i.data.get("representations")[0].get("name") + template_data["representation"] = rep + template_data["ext"] = rep + template_data["comment"] = None + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + filepath = os.path.normpath(template_filled) + self.log.info("Using published scene for render {}".format( + filepath)) + + if not os.path.exists(filepath): + self.log.error("published scene does not exist!") + raise + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + new_scene = os.path.splitext( + os.path.basename(filepath))[0] + orig_scene = os.path.splitext( + os.path.basename(context.data["currentFile"]))[0] + exp = instance.data.get("expectedFiles") + + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in exp[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( f.replace(orig_scene, new_scene) ) - instance.data["expectedFiles"] = [new_exp] - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) + new_exp[aov] = replaced_files + instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in exp: + new_exp.append( + f.replace(orig_scene, new_scene) + ) + instance.data["expectedFiles"] = [new_exp] + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) + # patch workfile is needed + if filepath not in patched_files: + patched_file = self._patch_workfile(filepath, patches) + patched_files.append(patched_file) all_instances = [] for result in context.data["results"]: @@ -868,10 +881,11 @@ def _get_arnold_export_payload(self, data): payload["JobInfo"].update(job_info_ext) payload["PluginInfo"].update(plugin_info_ext) - envs = [] - for k, v in payload["JobInfo"].items(): - if k.startswith("EnvironmentKeyValue"): - envs.append(v) + envs = [ + v + for k, v in payload["JobInfo"].items() + if k.startswith("EnvironmentKeyValue") + ] # add app name to environment envs.append( @@ -892,11 +906,8 @@ def _get_arnold_export_payload(self, data): envs.append( "OPENPYPE_ASS_EXPORT_STEP={}".format(1)) - i = 0 - for e in envs: + for i, e in enumerate(envs): payload["JobInfo"]["EnvironmentKeyValue{}".format(i)] = e - i += 1 - return payload def _get_vray_render_payload(self, data): @@ -1003,7 +1014,7 @@ def _requests_post(self, *args, **kwargs): """ if 'verify' not in kwargs: - kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa + kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) # add 10sec timeout before bailing out kwargs['timeout'] = 10 return requests.post(*args, **kwargs) @@ -1022,7 +1033,7 @@ def _requests_get(self, *args, **kwargs): """ if 'verify' not in kwargs: - kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa + kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) # add 10sec timeout before bailing out kwargs['timeout'] = 10 return requests.get(*args, **kwargs) @@ -1069,3 +1080,43 @@ def smart_replace(string, key_values): result = filename_zero.replace("\\", "/") return result + + def _patch_workfile(self, file, patches): + # type: (str, dict) -> [str, None] + """Patch Maya scene. + + This will take list of patches (lines to add) and apply them to + *published* Maya scene file (that is used later for rendering). + + Patches are dict with following structure:: + { + "name": "Name of patch", + "regex": "regex of line before patch", + "line": "line to insert" + } + + Args: + file (str): File to patch. + patches (dict): Dictionary defining patches. + + Returns: + str: Patched file path or None + + """ + if os.path.splitext(file)[1].lower() != ".ma" or not patches: + return None + + compiled_regex = [re.compile(p["regex"]) for p in patches] + with open(file, "r+") as pf: + scene_data = pf.readlines() + for ln, line in enumerate(scene_data): + for i, r in enumerate(compiled_regex): + if re.match(r, line): + scene_data.insert(ln + 1, patches[i]["line"]) + pf.seek(0) + pf.writelines(scene_data) + pf.truncate() + self.log.info( + "Applied {} patch to scene.".format( + patches[i]["name"])) + return file diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index 0f2da9f5b05..efeafbb1acb 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -45,7 +45,8 @@ "group": "none", "limit": [], "jobInfo": {}, - "pluginInfo": {} + "pluginInfo": {}, + "scene_patches": [] }, "NukeSubmitDeadline": { "enabled": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index 8e6a4b10e4b..53c6bf48c07 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -151,7 +151,7 @@ "type": "dict", "collapsible": true, "key": "MayaSubmitDeadline", - "label": "Submit maya job to deadline", + "label": "Submit Maya job to Deadline", "checkbox_key": "enabled", "children": [ { @@ -213,6 +213,31 @@ "type": "raw-json", "key": "pluginInfo", "label": "Additional PluginInfo data" + }, + { + "type": "list", + "key": "scene_patches", + "label": "Scene patches", + "required_keys": ["name", "regex", "line"], + "object_type": { + "type": "dict", + "children": [ + { + "key": "name", + "label": "Patch name", + "type": "text" + }, { + "key": "regex", + "label": "Patch regex", + "type": "text" + }, { + "key": "line", + "label": "Patch line", + "type": "text" + } + ] + + } } ] }, diff --git a/website/docs/admin_hosts_maya.md b/website/docs/admin_hosts_maya.md index 5e0aa153458..47447983b96 100644 --- a/website/docs/admin_hosts_maya.md +++ b/website/docs/admin_hosts_maya.md @@ -87,6 +87,26 @@ When you publish your model with top group named like `foo_GRP` it will fail. Bu All regexes used here are in Python variant. ::: +### Maya > Deadline submitter +This plugin provides connection between Maya and Deadline. It is using [Deadline Webservice](https://docs.thinkboxsoftware.com/products/deadline/10.0/1_User%20Manual/manual/web-service.html) to submit jobs to farm. +![Maya > Deadline Settings](assets/maya-admin_submit_maya_job_to_deadline.png) + +You can set various aspects of scene submission to farm with per-project settings in **Setting UI**. + + - **Optional** will mark sumission plugin optional + - **Active** will enable/disable plugin + - **Tile Assembler Plugin** will set what should be used to assemble tiles on Deadline. Either **Open Image IO** will be used +or Deadlines **Draft Tile Assembler**. + - **Use Published scene** enable to render from published scene instead of scene in work area. Rendering from published files is much safer. + - **Use Asset dependencies** will mark job pending on farm until asset dependencies are fulfilled - for example Deadline will wait for scene file to be synced to cloud, etc. + - **Group name** use specific Deadline group for the job. + - **Limit Groups** use these Deadline Limit groups for the job. + - **Additional `JobInfo` data** JSON of additional Deadline options that will be embedded in `JobInfo` part of the submission data. + - **Additional `PluginInfo` data** JSON of additional Deadline options that will be embedded in `PluginInfo` part of the submission data. + - **Scene patches** - configure mechanism to add additional lines to published Maya Ascii scene files before they are used for rendering. +This is useful to fix some specific renderer glitches and advanced hacking of Maya Scene files. `Patch name` is label for patch for easier orientation. +`Patch regex` is regex used to find line in file, after `Patch line` string is inserted. Note that you need to add line ending. + ## Custom Menu You can add your custom tools menu into Maya by extending definitions in **Maya -> Scripts Menu Definition**. ![Custom menu definition](assets/maya-admin_scriptsmenu.png) diff --git a/website/docs/assets/maya-admin_submit_maya_job_to_deadline.png b/website/docs/assets/maya-admin_submit_maya_job_to_deadline.png new file mode 100644 index 00000000000..56b720dc5de Binary files /dev/null and b/website/docs/assets/maya-admin_submit_maya_job_to_deadline.png differ