Skip to content
This repository has been archived by the owner on Sep 20, 2024. It is now read-only.

Deadline timeout and logging #2312

Merged
merged 5 commits into from
Nov 26, 2021
Merged

Deadline timeout and logging #2312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
df6a3fa
Added more logging
kalisp Nov 25, 2021
22f8c13
Added possibility for timeout as in other places
kalisp Nov 25, 2021
883c133
Add explicit higher timeout for cloud nodes
kalisp Nov 25, 2021
8a8f39d
Fixing typo
kalisp Nov 25, 2021
d76c572
Fixed label
kalisp Nov 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion igniter/tools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def validate_mongo_connection(cnx: str) -> (bool, str):
return False, "Not mongodb schema"

kwargs = {
"serverSelectionTimeoutMS": 2000
"serverSelectionTimeoutMS": os.environ.get("AVALON_TIMEOUT", 2000)
}
# Add certificate path if should be required
if should_add_certificate_path_to_mongo_url(cnx):
Expand Down
45 changes: 33 additions & 12 deletions vendor/deadline/custom/plugins/GlobalJobPreLoad.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ def inject_openpype_environment(deadlinePlugin):
job = deadlinePlugin.GetJob()
job = RepositoryUtils.GetJob(job.JobId, True) # invalidates cache

print("inject_openpype_environment start")
print(">>> Injecting OpenPype environments ...")
try:
print(">>> Getting OpenPype executable ...")
exe_list = job.GetJobExtraInfoKeyValue("openpype_executables")
openpype_app = FileUtils.SearchFileList(exe_list)
if openpype_app == "":
Expand All @@ -27,11 +28,13 @@ def inject_openpype_environment(deadlinePlugin):
"The path to the render executable can be configured " +
"from the Plugin Configuration in the Deadline Monitor.")

print("--- OpenPype executable: {}".format(openpype_app))

# tempfile.TemporaryFile cannot be used because of locking
export_url = os.path.join(tempfile.gettempdir(),
time.strftime('%Y%m%d%H%M%S'),
'env.json') # add HHMMSS + delete later
print("export_url {}".format(export_url))
print(">>> Temporary path: {}".format(export_url))

args = [
openpype_app,
Expand All @@ -55,41 +58,52 @@ def inject_openpype_environment(deadlinePlugin):
"AVALON_TASK, AVALON_APP_NAME"
raise RuntimeError(msg)

print("args:::{}".format(args))
if not os.environ.get("OPENPYPE_MONGO"):
print(">>> Missing OPENPYPE_MONGO env var, process won't work")

env = os.environ
env["OPENPYPE_HEADLESS_MODE"] = "1"
env["AVALON_TIMEOUT"] = "5000"

exit_code = subprocess.call(args, cwd=os.path.dirname(openpype_app))
if exit_code != 0:
raise RuntimeError("Publishing failed, check worker's log")
print(">>> Executing: {}".format(args))
std_output = subprocess.check_output(args,
cwd=os.path.dirname(openpype_app),
env=env)
print(">>> Process result {}".format(std_output))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now I am not sure if format() won't actually take care of it but as I reacall, output from this comand is bytes so it needs to be properly encoded to utf-8?

Suggested change
print(">>> Process result {}".format(std_output))
print(">>> Process result {}".format(std_output.decode("utf-8")))

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Its getting printed even without this, but the content length is imho limited.


print(">>> Loading file ...")
with open(export_url) as fp:
contents = json.load(fp)
for key, value in contents.items():
deadlinePlugin.SetProcessEnvironmentVariable(key, value)

print(">>> Removing temporary file")
os.remove(export_url)

print("inject_openpype_environment end")
except Exception:
print(">> Injection end.")
except Exception as e:
if hasattr(e, "output"):
print(">>> Exception {}".format(e.output))
import traceback
print(traceback.format_exc())
print("inject_openpype_environment failed")
print("!!! Injection failed.")
RepositoryUtils.FailJob(job)
raise


def inject_render_job_id(deadlinePlugin):
"""Inject dependency ids to publish process as env var for validation."""
print("inject_render_job_id start")
print(">>> Injecting render job id ...")
job = deadlinePlugin.GetJob()
job = RepositoryUtils.GetJob(job.JobId, True) # invalidates cache

dependency_ids = job.JobDependencyIDs
print("dependency_ids {}".format(dependency_ids))
print(">>> Dependency IDs: {}".format(dependency_ids))
render_job_ids = ",".join(dependency_ids)

deadlinePlugin.SetProcessEnvironmentVariable("RENDER_JOB_IDS",
render_job_ids)
print("inject_render_job_id end")
print(">>> Injection end.")


def pype_command_line(executable, arguments, workingDirectory):
Expand Down Expand Up @@ -133,10 +147,13 @@ def pype(deadlinePlugin):
deadlinePlugin: Deadline job plugin passed by Deadline

"""
print(">>> Getting job ...")
job = deadlinePlugin.GetJob()
# PYPE should be here, not OPENPYPE - backward compatibility!!
pype_metadata = job.GetJobEnvironmentKeyValue("PYPE_METADATA_FILE")
pype_python = job.GetJobEnvironmentKeyValue("PYPE_PYTHON_EXE")
print(">>> Having backward compatible env vars {}/{}".format(pype_metadata,
pype_python))
# test if it is pype publish job.
if pype_metadata:
pype_metadata = RepositoryUtils.CheckPathMapping(pype_metadata)
Expand All @@ -162,6 +179,8 @@ def pype(deadlinePlugin):


def __main__(deadlinePlugin):
print("*** GlobalJobPreload start ...")
print(">>> Getting job ...")
job = deadlinePlugin.GetJob()
job = RepositoryUtils.GetJob(job.JobId, True) # invalidates cache

Expand All @@ -170,6 +189,8 @@ def __main__(deadlinePlugin):
openpype_publish_job = \
job.GetJobEnvironmentKeyValue('OPENPYPE_PUBLISH_JOB') or '0'

print("--- Job type - render {}".format(openpype_render_job))
print("--- Job type - publish {}".format(openpype_publish_job))
if openpype_publish_job == '1' and openpype_render_job == '1':
raise RuntimeError("Misconfiguration. Job couldn't be both " +
"render and publish.")
Expand Down