#636 - Deadline Output Folder

Pre-calculate final publish folder to add it to right mouse click option
for easier checking in Deadline Monitor

pype/plugins/global/publish/submit_publish_job.py

@@ -4,7 +4,8 @@
 import os
 import json
 import re
-from copy import copy
+from copy import copy, deepcopy
+import pype.api
 
 from avalon import api, io
 from avalon.vendor import requests, clique
@@ -42,40 +43,6 @@ def _get_script(path):
  return str(path)
 
 
-def get_latest_version(asset_name, subset_name, family):
- """Retrieve latest files concerning extendFrame feature."""
- # Get asset
- asset_name = io.find_one(
- {"type": "asset", "name": asset_name}, projection={"name": True}
- )
-
- subset = io.find_one(
- {"type": "subset", "name": subset_name, "parent": asset_name["_id"]},
- projection={"_id": True, "name": True},
- )
-
- # Check if subsets actually exists (pre-run check)
- assert subset, "No subsets found, please publish with `extendFrames` off"
-
- # Get version
- version_projection = {
- "name": True,
- "data.startFrame": True,
- "data.endFrame": True,
- "parent": True,
- }
-
- version = io.find_one(
- {"type": "version", "parent": subset["_id"], "data.families": family},
- projection=version_projection,
- sort=[("name", -1)],
- )
-
- assert version, "No version found, this is a bug"
-
- return version
-
-
 def get_resources(version, extension=None):
  """Get the files from the specific version."""
  query = {"type": "representation", "parent": version["_id"]}
@@ -250,7 +217,19 @@ def _submit_deadline_post_job(self, instance, job, instances):
  subset = data["subset"]
  job_name = "Publish - {subset}".format(subset=subset)
 
- output_dir = instance.data["outputDir"]
+ # instance.data.get("subset") != instances[0]["subset"]
+ # 'Main' vs 'renderMain'
+ override_version = None
+ instance_version = instance.data.get("version") # take this if exists
+ if instance_version != 1:
+ override_version = instance_version
+ output_dir = self._get_publish_folder(instance.context.data['anatomy'],
+ deepcopy(
+ instance.data["anatomyData"]),
+ instance.data.get("asset"),
+ instances[0]["subset"],
+ 'render',
+ override_version)
 
  # Generate the payload for Deadline submission
  payload = {
@@ -322,7 +301,6 @@ def _submit_deadline_post_job(self, instance, job, instances):
  payload["JobInfo"].pop("SecondaryPool", None)
 
  self.log.info("Submitting Deadline job ...")
- # self.log.info(json.dumps(payload, indent=4, sort_keys=True))
 
  url = "{}/api/jobs".format(self.DEADLINE_REST_URL)
  response = requests.post(url, json=payload, timeout=10)
@@ -349,9 +327,8 @@ def _copy_extend_frames(self, instance, representation):
 
  # get latest version of subset
  # this will stop if subset wasn't published yet
- version = get_latest_version(
- instance.data.get("asset"),
- instance.data.get("subset"), "render")
+ version = pype.api.get_latest_version(instance.data.get("asset"),
+ instance.data.get("subset"))
  # get its files based on extension
  subset_resources = get_resources(version, representation.get("ext"))
  r_col, _ = clique.assemble(subset_resources)
@@ -732,8 +709,7 @@ def process(self, instance):
  "resolutionHeight": data.get("resolutionHeight", 1080),
  "multipartExr": data.get("multipartExr", False),
  "jobBatchName": data.get("jobBatchName", ""),
- "review": data.get("review", True),
- "audio": data.get("audio", [])
+ "review": data.get("review", True)
  }
 
  if "prerender" in instance.data["families"]:
@@ -742,7 +718,7 @@ def process(self, instance):
  "families": []})
 
  # skip locking version if we are creating v01
- instance_version = instance.data.get("version")
+ instance_version = instance.data.get("version") # take this if exists
  if instance_version != 1:
  instance_skeleton_data["version"] = instance_version
 
@@ -998,11 +974,9 @@ def _extend_frames(self, asset, subset, start, end):
  prev_start = None
  prev_end = None
 
- version = get_latest_version(
- asset_name=asset,
- subset_name=subset,
- family='render'
- )
+ version = pype.api.get_latest_version(asset_name=asset,
+ subset_name=subset
+ )
 
  # Set prev start / end frames for comparison
  if not prev_start and not prev_end:
@@ -1018,3 +992,58 @@ def _extend_frames(self, asset, subset, start, end):
  )
 
  return updated_start, updated_end
+
+ def _get_publish_folder(self, anatomy, template_data,
+ asset, subset,
+ family='render', version=None):
+ """
+ Extracted logic to pre-calculate real publish folder, which is
+ calculated in IntegrateNew inside of Deadline process.
+ This should match logic in:
+ 'collect_anatomy_instance_data' - to
+ get correct anatomy, family, version for subset and
+ 'collect_resources_path'
+ get publish_path
+
+ Args:
+ anatomy (pypeapp.lib.anatomy.Anatomy):
+ template_data (dict): pre-calculated collected data for process
+ asset (string): asset name
+ subset (string): subset name (actually group name of subset)
+ family (string): for current deadline process it's always 'render'
+ TODO - for generic use family needs to be dynamically
+ calculated like IntegrateNew does
+ version (int): override version from instance if exists
+
+ Returns:
+ (string): publish folder where rendered and published files will
+ be stored
+ based on 'publish' template
+ """
+ if not version:
+ version = pype.api.get_latest_version(asset, subset)
+ if version:
+ version = int(version["name"]) + 1
+
+ template_data["subset"] = subset
+ template_data["family"] = "render"
+ template_data["version"] = version
+
+ anatomy_filled = anatomy.format(template_data)
+
+ if "folder" in anatomy.templates["publish"]:
+ publish_folder = anatomy_filled["publish"]["folder"]
+ else:
+ # solve deprecated situation when `folder` key is not underneath
+ # `publish` anatomy
+ project_name = api.Session["AVALON_PROJECT"]
+ self.log.warning((
+ "Deprecation warning: Anatomy does not have set `folder`"
+ " key underneath `publish` (in global of for project `{}`)."
+ ).format(project_name))
+
+ file_path = anatomy_filled["publish"]["path"]
+ # Directory
+ publish_folder = os.path.dirname(file_path)
+
+ return publish_folder