Merge pull request #41 from pypeclub/fix/deadline_cleanup

Deadline - cleanup, add documentation

openpype/lib/__init__.py

@@ -24,7 +24,7 @@
  compose_url,
  get_default_components,
  validate_mongo_connection,
- PypeMongoConnection
+ OpenPypeMongoConnection
 )
 from .anatomy import (
  merge_dict,
@@ -213,7 +213,7 @@
  "compose_url",
  "get_default_components",
  "validate_mongo_connection",
- "PypeMongoConnection",
+ "OpenPypeMongoConnection",
 
  "IniSettingRegistry",
  "JSONSettingRegistry",

openpype/lib/log.py

@@ -28,7 +28,7 @@
 from .mongo import (
  MongoEnvNotSet,
  decompose_url,
- PypeMongoConnection
+ OpenPypeMongoConnection
 )
 try:
  import log4mongo
@@ -203,7 +203,7 @@ class PypeLogger:
  log_mongo_url_components = None
 
  # Database name in Mongo
- log_database_name = "pype"
+ log_database_name = os.environ["OPENPYPE_DATABASE_NAME"]
  # Collection name under database in Mongo
  log_collection_name = "logs"
 
@@ -470,7 +470,7 @@ def get_log_mongo_connection(cls):
  if not cls.initialized:
  cls.initialize()
 
- return PypeMongoConnection.get_mongo_client(cls.log_mongo_url)
+ return OpenPypeMongoConnection.get_mongo_client(cls.log_mongo_url)
 
 
 def timeit(method):

openpype/lib/mongo.py

@@ -129,13 +129,13 @@ def validate_mongo_connection(mongo_uri):
  client.close()
 
 
-class PypeMongoConnection:
+class OpenPypeMongoConnection:
  """Singleton MongoDB connection.
 
  Keeps MongoDB connections by url.
  """
  mongo_clients = {}
- log = logging.getLogger("PypeMongoConnection")
+ log = logging.getLogger("OpenPypeMongoConnection")
 
  @staticmethod
  def get_default_mongo_url():

openpype/modules/deadline/deadline_module.py

@@ -13,7 +13,7 @@ def initialize(self, modules_settings):
  self.deadline_url = deadline_settings["DEADLINE_REST_URL"]
 
  def get_global_environments(self):
- """Deadline global environments for pype implementation."""
+ """Deadline global environments for OpenPype implementation."""
  return {
  "DEADLINE_REST_URL": self.deadline_url
  }

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -98,7 +98,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
  label = "Submit image sequence jobs to Deadline or Muster"
  order = pyblish.api.IntegratorOrder + 0.2
  icon = "tractor"
- deadline_plugin = "Pype"
+ deadline_plugin = "OpenPype"
 
  hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"]
 

openpype/modules/ftrack/ftrack_server/event_server_cli.py

@@ -16,7 +16,7 @@
 import pymongo
 from openpype.lib import (
  get_pype_execute_args,
- PypeMongoConnection
+ OpenPypeMongoConnection
 )
 from openpype.modules.ftrack import FTRACK_MODULE_DIR
 from openpype.modules.ftrack.lib import (
@@ -181,7 +181,7 @@ def main_loop(ftrack_url):
 
  os.environ["FTRACK_EVENT_SUB_ID"] = str(uuid.uuid1())
 
- mongo_uri = PypeMongoConnection.get_default_mongo_url()
+ mongo_uri = OpenPypeMongoConnection.get_default_mongo_url()
 
  # Current file
  scripts_dir = os.path.join(FTRACK_MODULE_DIR, "scripts")

openpype/modules/ftrack/ftrack_server/lib.py

@@ -24,7 +24,7 @@
  from ftrack_api._weakref import WeakMethod
 from openpype.modules.ftrack.lib import get_ftrack_event_mongo_info
 
-from openpype.lib import PypeMongoConnection
+from openpype.lib import OpenPypeMongoConnection
 from openpype.api import Logger
 
 TOPIC_STATUS_SERVER = "pype.event.server.status"
@@ -144,14 +144,14 @@ def __init__(self, *args, **kwargs):
  def prepare_dbcon(self):
  try:
  database_name, collection_name = get_ftrack_event_mongo_info()
- mongo_client = PypeMongoConnection.get_mongo_client()
+ mongo_client = OpenPypeMongoConnection.get_mongo_client()
  self.dbcon = mongo_client[database_name][collection_name]
  self.mongo_client = mongo_client
 
  except pymongo.errors.AutoReconnect:
  self.pypelog.error((
  "Mongo server \"{}\" is not responding, exiting."
- ).format(PypeMongoConnection.get_default_mongo_url()))
+ ).format(OpenPypeMongoConnection.get_default_mongo_url()))
  sys.exit(0)
 
  except pymongo.errors.OperationFailure:

openpype/modules/ftrack/lib/settings.py

@@ -1,8 +1,6 @@
+import os
 from openpype.api import get_system_settings
 
-PYPE_DATABASE_NAME = "pype"
-
-
 def get_ftrack_settings():
  return get_system_settings()["modules"]["ftrack"]
 
@@ -13,6 +11,6 @@ def get_ftrack_url_from_settings():
 
 def get_ftrack_event_mongo_info():
  ftrack_settings = get_ftrack_settings()
- database_name = PYPE_DATABASE_NAME
+ database_name = os.environ["OPENPYPE_DATABASE_NAME"]
  collection_name = "ftrack_events"
  return database_name, collection_name

openpype/modules/ftrack/scripts/sub_event_storer.py

@@ -14,7 +14,7 @@
  TOPIC_STATUS_SERVER_RESULT
 )
 from openpype.modules.ftrack.lib import get_ftrack_event_mongo_info
-from openpype.lib import PypeMongoConnection
+from openpype.lib import OpenPypeMongoConnection
 from openpype.api import Logger
 
 log = Logger.get_logger("Event storer")
@@ -35,11 +35,11 @@ class SessionFactory:
 def install_db():
  global dbcon
  try:
- mongo_client = PypeMongoConnection.get_mongo_client()
+ mongo_client = OpenPypeMongoConnection.get_mongo_client()
  dbcon = mongo_client[database_name][collection_name]
  except pymongo.errors.AutoReconnect:
  log.error("Mongo server \"{}\" is not responding, exiting.".format(
- PypeMongoConnection.get_default_mongo_url()
+ OpenPypeMongoConnection.get_default_mongo_url()
  ))
  sys.exit(0)
 

openpype/settings/handlers.py

@@ -170,18 +170,18 @@ class MongoSettingsHandler(SettingsHandler):
 
  def __init__(self):
  # Get mongo connection
- from openpype.lib import PypeMongoConnection
+ from openpype.lib import OpenPypeMongoConnection
  from avalon.api import AvalonMongoDB
 
- settings_collection = PypeMongoConnection.get_mongo_client()
+ settings_collection = OpenPypeMongoConnection.get_mongo_client()
 
  self._anatomy_keys = None
  self._attribute_keys = None
  # TODO prepare version of pype
  # - pype version should define how are settings saved and loaded
 
+ database_name = os.environ["OPENPYPE_DATABASE_NAME"]
  # TODO modify to not use hardcoded keys
- database_name = "pype"
  collection_name = "settings"
 
  self.settings_collection = settings_collection
@@ -541,19 +541,19 @@ class MongoLocalSettingsHandler(LocalSettingsHandler):
  def __init__(self, local_site_id=None):
  # Get mongo connection
  from openpype.lib import (
- PypeMongoConnection,
+ OpenPypeMongoConnection,
  get_local_site_id
  )
 
  if local_site_id is None:
  local_site_id = get_local_site_id()
- settings_collection = PypeMongoConnection.get_mongo_client()
+ settings_collection = OpenPypeMongoConnection.get_mongo_client()
 
  # TODO prepare version of pype
  # - pype version should define how are settings saved and loaded
 
+ database_name = os.environ["OPENPYPE_DATABASE_NAME"]
  # TODO modify to not use hardcoded keys
- database_name = "pype"
  collection_name = "settings"
 
  self.settings_collection = settings_collection

start.py

@@ -561,6 +561,7 @@ def boot():
  sys.exit(1)
 
  os.environ["OPENPYPE_MONGO"] = openpype_mongo
+ os.environ["OPENPYPE_DATABASE_NAME"] = "openpype" # name of Pype database
 
  # ------------------------------------------------------------------------
  # Set environments - load OpenPype path from database (if set)

vendor/deadline/custom/events/OpenPype/OpenPype.param

@@ -28,8 +28,8 @@ Default=DEBUG
 Description=Logging level where printing will start.
 
 [OpenPypeExecutable]
-Type=MultiLineMultiFolder
-Label=Path to OpenPype executable dir
+Type=multilinemultifilename
+Label=Path to OpenPype executable
 Category=Job Plugins
 CategoryOrder=1
 CategoryIndex=1

vendor/deadline/custom/plugins/OpenPype/OpenPype.options

@@ -1,13 +1,3 @@
-[ScriptFile]
-Type=filename
-Label=Script File
-Category=Python Options
-CategoryOrder=0
-Index=0
-Description=The script file to be executed.
-Required=false
-DisableIfBlank=true
-
 [Arguments]
 Type=string
 Label=Arguments
@@ -18,24 +8,3 @@ Description=The arguments to pass to the script. If no arguments are required, l
 Required=false
 DisableIfBlank=true
 
-[Version]
-Type=enum
-Values=3.0
-Label=Version
-Category=Python Options
-CategoryOrder=0
-Index=2
-Description=The version of Python to use.
-Required=false
-DisableIfBlank=true
-
-[SingleFramesOnly]
-Type=boolean
-Label=Single Frames Only
-Category=Job Options
-CategoryOrder=1
-Index=0
-Description=If enabled, the plugin will only render one frame at a time even if a single task contains a chunk of frames.
-Required=true
-DisableIfBlank=true
-Default=false

vendor/deadline/custom/plugins/OpenPype/OpenPype.param

@@ -7,18 +7,9 @@ Index=0
 Default=OpenPype Plugin for Deadline
 Description=Not configurable
 
-[ConcurrentTasks]
-Type=label
-Label=ConcurrentTasks
-Category=About Plugin
-CategoryOrder=-1
-Index=0
-Default=True
-Description=Not configurable
-
-[OpenPype_Executable_3_0]
+[OpenPypeExecutable]
 Type=multilinemultifilename
-Label=OpenPype 3.0 Executable
+Label=OpenPype Executable
 Category=OpenPype Executables
 CategoryOrder=0
 Index=0

vendor/deadline/custom/plugins/OpenPype/OpenPype.py

@@ -53,14 +53,11 @@ def InitializeProcess(self):
  ".*Progress: (\d+)%.*").HandleCallback += self.HandleProgress
 
  def RenderExecutable(self):
- version = self.GetPluginInfoEntry("Version")
-
- exeList = self.GetConfigEntry(
- "OpenPype_Executable_" + version.replace(".", "_"))
+ exeList = self.GetConfigEntry("OpenPypeExecutable")
  exe = FileUtils.SearchFileList(exeList)
  if exe == "":
  self.FailRender(
- "OpenPype " + version + " executable was not found " +
+ "OpenPype executable was not found " +
  "in the semicolon separated list \"" + exeList + "\". " +
  "The path to the render executable can be configured " +
  "from the Plugin Configuration in the Deadline Monitor.")

vendor/deadline/readme.md

@@ -1,3 +1,32 @@
 ## OpenPype Deadline repository overlay
 
- This directory is overlay for Deadline repository. It means that you can copy whole hierarchy to Deadline repository and it should work.
+ This directory is overlay for Deadline repository. 
+ It means that you can copy whole hierarchy to Deadline repository and it should work.
+
+ Logic:
+ -----
+ Event
+ -----
+ For each rendering job OpenPype event is triggered, it stores path to OpenPype
+ executable (needs to be configured on `Deadline's Configure Events > OpenPype`) 
+ job's extra key 'openpype_executables'.
+
+ This value is used by `GlobalJobPreLoad` to call that executable to pull
+ environment's variables which are needed to add to ALL plugins process environments.
+ These env. vars are injected into rendering process.
+
+ Event is necessary here as a middle man to allow configuring location of executable
+ which is ONLY then used by `GlobalJobPreLoad` (which doesnt have any user facing
+ configuration at all).
+
+ `GlobalJobPreLoad` is triggered before each job, it contains backward compatible
+ logic to not modify old Pype2 or not OpenPype triggered jobs.
+
+ Plugin
+ ------
+ For each publishing job `OpenPypeDeadlinePlugin` is called, which calls 
+ configured location of OpenPype executable (needs to be configured in 
+ `Deadline's Configure Plugins > OpenPype`) 
+ and triggers command.
+
+