Tests: add support for deadline for automatic tests

igniter/bootstrap_repos.py

@@ -61,7 +61,8 @@ class OpenPypeVersion(semver.VersionInfo):
 
  """
  path = None
- _VERSION_REGEX = re.compile(r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?") # noqa: E501
+ # this should match any string complying with https://semver.org/
+ _VERSION_REGEX = re.compile(r"(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>[a-zA-Z\d\-.]*))?(?:\+(?P<buildmetadata>[a-zA-Z\d\-.]*))?") # noqa: E501
  _installed_version = None
 
  def __init__(self, *args, **kwargs):
@@ -154,6 +155,8 @@ def version_in_str(string: str) -> Union[None, OpenPypeVersion]:
  OpenPypeVersion: of detected or None.
 
  """
+ # strip .zip ext if present
+ string = re.sub(r"\.zip$", "", string, flags=re.IGNORECASE)
  m = re.search(OpenPypeVersion._VERSION_REGEX, string)
  if not m:
  return None

igniter/tools.py

@@ -153,7 +153,8 @@ def get_openpype_global_settings(url: str) -> dict:
  # Create mongo connection
  client = MongoClient(url, **kwargs)
  # Access settings collection
- col = client["openpype"]["settings"]
+ openpype_db = os.environ.get("OPENPYPE_DATABASE_NAME") or "openpype"
+ col = client[openpype_db]["settings"]
  # Query global settings
  global_settings = col.find_one({"type": "global_settings"}) or {}
  # Close Mongo connection

openpype/cli.py

@@ -22,7 +22,9 @@
 @click.option("--debug", is_flag=True, expose_value=False,
  help="Enable debug")
 @click.option("--verbose", expose_value=False,
- help="Change OpenPype log level (debug - critical or 0-50)")
+ help=("Change OpenPype log level (debug - critical or 0-50)"))
+@click.option("--automatic-tests", is_flag=True, expose_value=False,
+ help=("Run in automatic tests mode"))
 def main(ctx):
  """Pype is main command serving as entry point to pipeline system.
 

openpype/hosts/aftereffects/api/lib.py

@@ -13,6 +13,7 @@
 from openpype.modules import ModulesManager
 
 from openpype.tools.utils import host_tools
+from openpype.tests.lib import is_in_tests
 from .launch_logic import ProcessLauncher, get_stub
 
 log = logging.getLogger(__name__)
@@ -47,7 +48,7 @@ def main(*subprocess_args):
  webpublisher_addon.headless_publish,
  log,
  "CloseAE",
- os.environ.get("IS_TEST")
+ is_in_tests()
  )
  )
 

openpype/hosts/nuke/api/plugin.py

@@ -427,6 +427,8 @@ def save_file(self):
  # create nk path
  path = os.path.splitext(self.path)[0] + ".nk"
  # save file to the path
+ if not os.path.exists(os.path.dirname(path)):
+ os.makedirs(os.path.dirname(path))
  shutil.copyfile(self.instance.context.data["currentFile"], path)
 
  self.log.info("Nodes exported...")

openpype/hosts/photoshop/api/lib.py

@@ -9,6 +9,7 @@
 from openpype.modules import ModulesManager
 from openpype.pipeline import install_host
 from openpype.tools.utils import host_tools
+from openpype.tests.lib import is_in_tests
 
 from .launch_logic import ProcessLauncher, stub
 
@@ -42,7 +43,7 @@ def main(*subprocess_args):
  webpublisher_addon.headless_publish,
  log,
  "ClosePS",
- os.environ.get("IS_TEST")
+ is_in_tests()
  )
  elif env_value_to_bool("AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH",
  default=True):

openpype/hosts/photoshop/plugins/publish/collect_batch_data.py

@@ -22,6 +22,7 @@
  get_batch_asset_task_info,
  parse_json
 )
+from openpype.tests.lib import is_in_tests
 
 
 class CollectBatchData(pyblish.api.ContextPlugin):
@@ -39,7 +40,7 @@ class CollectBatchData(pyblish.api.ContextPlugin):
  def process(self, context):
  self.log.info("CollectBatchData")
  batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
- if os.environ.get("IS_TEST"):
+ if is_in_tests():
  self.log.debug("Automatic testing, no batch data, skipping")
  return
 

openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py

@@ -6,6 +6,7 @@
 from openpype.lib import prepare_template_data
 from openpype.hosts.photoshop import api as photoshop
 from openpype.settings import get_project_settings
+from openpype.tests.lib import is_in_tests
 
 
 class CollectColorCodedInstances(pyblish.api.ContextPlugin):
@@ -46,7 +47,7 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin):
  def process(self, context):
  self.log.info("CollectColorCodedInstances")
  batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
- if (os.environ.get("IS_TEST") and
+ if (is_in_tests() and
  (not batch_dir or not os.path.exists(batch_dir))):
  self.log.debug("Automatic testing, no batch data, skipping")
  return

openpype/hosts/tvpaint/plugins/publish/collect_instance_frames.py

@@ -6,7 +6,7 @@ class CollectOutputFrameRange(pyblish.api.ContextPlugin):
 
  When instances are collected context does not contain `frameStart` and
  `frameEnd` keys yet. They are collected in global plugin
- `CollectAvalonEntities`.
+ `CollectContextEntities`.
  """
  label = "Collect output frame range"
  order = pyblish.api.CollectorOrder

openpype/hosts/tvpaint/plugins/publish/validate_marks.py

@@ -39,7 +39,7 @@ class ValidateMarks(pyblish.api.ContextPlugin):
  def get_expected_data(context):
  scene_mark_in = context.data["sceneMarkIn"]
 
- # Data collected in `CollectAvalonEntities`
+ # Data collected in `CollectContextEntities`
  frame_end = context.data["frameEnd"]
  frame_start = context.data["frameStart"]
  handle_start = context.data["handleStart"]

openpype/hosts/webpublisher/plugins/publish/validate_tvpaint_workfile_data.py

@@ -13,7 +13,7 @@ class ValidateWorkfileData(pyblish.api.ContextPlugin):
  targets = ["tvpaint_worker"]
 
  def process(self, context):
- # Data collected in `CollectAvalonEntities`
+ # Data collected in `CollectContextEntities`
  frame_start = context.data["frameStart"]
  frame_end = context.data["frameEnd"]
  handle_start = context.data["handleStart"]

openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py

@@ -2,6 +2,7 @@
 import attr
 import getpass
 import pyblish.api
+from datetime import datetime
 
 from openpype.lib import (
  env_value_to_bool,
@@ -10,6 +11,7 @@
 from openpype.pipeline import legacy_io
 from openpype_modules.deadline import abstract_submit_deadline
 from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
+from openpype.tests.lib import is_in_tests
 
 
 @attr.s
@@ -48,9 +50,11 @@ def get_job_info(self):
 
  context = self._instance.context
 
+ batch_name = os.path.basename(self._instance.data["source"])
+ if is_in_tests():
+ batch_name += datetime.now().strftime("%d%m%Y%H%M%S")
  dln_job_info.Name = self._instance.data["name"]
- dln_job_info.BatchName = os.path.basename(self._instance.
- data["source"])
+ dln_job_info.BatchName = batch_name
  dln_job_info.Plugin = "AfterEffects"
  dln_job_info.UserName = context.data.get(
  "deadlineUser", getpass.getuser())
@@ -83,7 +87,8 @@ def get_job_info(self):
  "AVALON_APP_NAME",
  "OPENPYPE_DEV",
  "OPENPYPE_LOG_NO_COLORS",
- "OPENPYPE_VERSION"
+ "OPENPYPE_VERSION",
+ "IS_TEST"
  ]
  # Add mongo url if it's enabled
  if self._instance.context.data.get("deadlinePassMongoUrl"):

openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py

@@ -5,13 +5,15 @@
 from collections import OrderedDict
 from zipfile import ZipFile, is_zipfile
 import re
+from datetime import datetime
 
 import attr
 import pyblish.api
 
 from openpype.pipeline import legacy_io
 from openpype_modules.deadline import abstract_submit_deadline
 from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
+from openpype.tests.lib import is_in_tests
 
 
 class _ZipFile(ZipFile):
@@ -261,7 +263,10 @@ def get_job_info(self):
  job_info.Pool = self._instance.data.get("primaryPool")
  job_info.SecondaryPool = self._instance.data.get("secondaryPool")
  job_info.ChunkSize = self.chunk_size
- job_info.BatchName = os.path.basename(self._instance.data["source"])
+ batch_name = os.path.basename(self._instance.data["source"])
+ if is_in_tests:
+ batch_name += datetime.now().strftime("%d%m%Y%H%M%S")
+ job_info.BatchName = batch_name
  job_info.Department = self.department
  job_info.Group = self.group
 
@@ -275,7 +280,8 @@ def get_job_info(self):
  "AVALON_APP_NAME",
  "OPENPYPE_DEV",
  "OPENPYPE_LOG_NO_COLORS",
- "OPENPYPE_VERSION"
+ "OPENPYPE_VERSION",
+ "IS_TEST"
  ]
  # Add mongo url if it's enabled
  if self._instance.context.data.get("deadlinePassMongoUrl"):

openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py

@@ -1,12 +1,14 @@
 import os
 import json
+from datetime import datetime
 
 import requests
 import hou
 
 import pyblish.api
 
 from openpype.pipeline import legacy_io
+from openpype.tests.lib import is_in_tests
 
 
 class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin):
@@ -60,6 +62,8 @@ def process(self, context):
 
  job_name = "{scene} [PUBLISH]".format(scene=scenename)
  batch_name = "{code} - {scene}".format(code=code, scene=scenename)
+ if is_in_tests():
+ batch_name += datetime.now().strftime("%d%m%Y%H%M%S")
  deadline_user = "roy" # todo: get deadline user dynamically
 
  # Get only major.minor version of Houdini, ignore patch version

openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py

@@ -1,13 +1,15 @@
 import os
 import json
 import getpass
+from datetime import datetime
 
 import requests
 import pyblish.api
 
 # import hou ???
 
 from openpype.pipeline import legacy_io
+from openpype.tests.lib import is_in_tests
 
 
 class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin):
@@ -45,6 +47,9 @@ def process(self, instance):
  if code:
  batch_name = "{0} - {1}".format(code, batch_name)
 
+ if is_in_tests():
+ batch_name += datetime.now().strftime("%d%m%Y%H%M%S")
+
  # Output driver to render
  driver = instance[0]
 

openpype/modules/deadline/plugins/publish/submit_maya_deadline.py

@@ -37,6 +37,7 @@
 
 from openpype_modules.deadline import abstract_submit_deadline
 from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
+from openpype.tests.lib import is_in_tests
 
 
 def _validate_deadline_bool_value(instance, attribute, value):
@@ -121,6 +122,9 @@ def get_job_info(self):
  src_filepath = context.data["currentFile"]
  src_filename = os.path.basename(src_filepath)
 
+ if is_in_tests():
+ src_filename += datetime.now().strftime("%d%m%Y%H%M%S")
+
  job_info.Name = "%s - %s" % (src_filename, instance.name)
  job_info.BatchName = src_filename
  job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch")
@@ -161,7 +165,8 @@ def get_job_info(self):
  "AVALON_TASK",
  "AVALON_APP_NAME",
  "OPENPYPE_DEV",
- "OPENPYPE_VERSION"
+ "OPENPYPE_VERSION",
+ "IS_TEST"
  ]
  # Add mongo url if it's enabled
  if self._instance.context.data.get("deadlinePassMongoUrl"):

openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py

@@ -1,10 +1,12 @@
 import os
 import requests
+from datetime import datetime
 
 from maya import cmds
 
 from openpype.pipeline import legacy_io, PublishXmlValidationError
 from openpype.settings import get_project_settings
+from openpype.tests.lib import is_in_tests
 
 import pyblish.api
 
@@ -57,6 +59,8 @@ def process(self, instance):
  job_name = "{scene} [PUBLISH]".format(scene=scenename)
  batch_name = "{code} - {scene}".format(code=project_name,
  scene=scenename)
+ if is_in_tests():
+ batch_name += datetime.now().strftime("%d%m%Y%H%M%S")
 
  # Generate the payload for Deadline submission
  payload = {

openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py

@@ -2,12 +2,14 @@
 import re
 import json
 import getpass
+from datetime import datetime
 
 import requests
 import pyblish.api
 
 import nuke
 from openpype.pipeline import legacy_io
+from openpype.tests.lib import is_in_tests
 
 
 class NukeSubmitDeadline(pyblish.api.InstancePlugin):
@@ -141,8 +143,11 @@ def payload_submit(
  responce_data=None
  ):
  render_dir = os.path.normpath(os.path.dirname(render_path))
- script_name = os.path.basename(script_path)
- jobname = "%s - %s" % (script_name, instance.name)
+ batch_name = os.path.basename(script_path)
+ jobname = "%s - %s" % (batch_name, instance.name)
+ if is_in_tests():
+ batch_name += datetime.now().strftime("%d%m%Y%H%M%S")
+
 
  output_filename_0 = self.preview_fname(render_path)
 
@@ -176,7 +181,7 @@ def payload_submit(
  payload = {
  "JobInfo": {
  # Top-level group name
- "BatchName": script_name,
+ "BatchName": batch_name,
 
  # Asset dependency to wait for at least the scene file to sync.
  # "AssetDependency0": script_path,