Merge pull request #2222 from pypeclub/feature/separate_webpublisher_…

…logic

Webpublisher: Separate webpublisher logic

openpype/cli.py

@@ -166,34 +166,35 @@ def publish(debug, paths, targets):
 @click.option("-p", "--project", help="Project")
 @click.option("-t", "--targets", help="Targets", default=None,
  multiple=True)
-def remotepublishfromapp(debug, project, path, host, targets=None, user=None):
+def remotepublishfromapp(debug, project, path, host, user=None, targets=None):
  """Start CLI publishing.
 
  Publish collects json from paths provided as an argument.
  More than one path is allowed.
  """
  if debug:
  os.environ['OPENPYPE_DEBUG'] = '3'
- PypeCommands.remotepublishfromapp(project, path, host, user,
- targets=targets)
+ PypeCommands.remotepublishfromapp(
+ project, path, host, user, targets=targets
+ )
+
 
 @main.command()
 @click.argument("path")
 @click.option("-d", "--debug", is_flag=True, help="Print debug messages")
-@click.option("-h", "--host", help="Host")
 @click.option("-u", "--user", help="User email address")
 @click.option("-p", "--project", help="Project")
 @click.option("-t", "--targets", help="Targets", default=None,
  multiple=True)
-def remotepublish(debug, project, path, host, targets=None, user=None):
+def remotepublish(debug, project, path, user=None, targets=None):
  """Start CLI publishing.
 
  Publish collects json from paths provided as an argument.
  More than one path is allowed.
  """
  if debug:
  os.environ['OPENPYPE_DEBUG'] = '3'
- PypeCommands.remotepublish(project, path, host, user, targets=targets)
+ PypeCommands.remotepublish(project, path, user, targets=targets)
 
 
 @main.command()

openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py

@@ -0,0 +1,84 @@
+"""Loads batch context from json and continues in publish process.
+
+Provides:
+ context -> Loaded batch file.
+"""
+
+import os
+
+import pyblish.api
+from avalon import io
+from openpype.lib.plugin_tools import (
+ parse_json,
+ get_batch_asset_task_info
+)
+from openpype.lib.remote_publish import get_webpublish_conn
+
+
+class CollectBatchData(pyblish.api.ContextPlugin):
+ """Collect batch data from json stored in 'OPENPYPE_PUBLISH_DATA' env dir.
+
+ The directory must contain 'manifest.json' file where batch data should be
+ stored.
+ """
+ # must be really early, context values are only in json file
+ order = pyblish.api.CollectorOrder - 0.495
+ label = "Collect batch data"
+ host = ["webpublisher"]
+
+ def process(self, context):
+ batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
+
+ assert batch_dir, (
+ "Missing `OPENPYPE_PUBLISH_DATA`")
+
+ assert os.path.exists(batch_dir), \
+ "Folder {} doesn't exist".format(batch_dir)
+
+ project_name = os.environ.get("AVALON_PROJECT")
+ if project_name is None:
+ raise AssertionError(
+ "Environment `AVALON_PROJECT` was not found."
+ "Could not set project `root` which may cause issues."
+ )
+
+ batch_data = parse_json(os.path.join(batch_dir, "manifest.json"))
+
+ context.data["batchDir"] = batch_dir
+ context.data["batchData"] = batch_data
+
+ asset_name, task_name, task_type = get_batch_asset_task_info(
+ batch_data["context"]
+ )
+
+ os.environ["AVALON_ASSET"] = asset_name
+ io.Session["AVALON_ASSET"] = asset_name
+ os.environ["AVALON_TASK"] = task_name
+ io.Session["AVALON_TASK"] = task_name
+
+ context.data["asset"] = asset_name
+ context.data["task"] = task_name
+ context.data["taskType"] = task_type
+
+ self._set_ctx_path(batch_data)
+
+ def _set_ctx_path(self, batch_data):
+ dbcon = get_webpublish_conn()
+
+ batch_id = batch_data["batch"]
+ ctx_path = batch_data["context"]["path"]
+ self.log.info("ctx_path: {}".format(ctx_path))
+ self.log.info("batch_id: {}".format(batch_id))
+ if ctx_path and batch_id:
+ self.log.info("Updating log record")
+ dbcon.update_one(
+ {
+ "batch_id": batch_id,
+ "status": "in_progress"
+ },
+ {
+ "$set": {
+ "path": ctx_path
+ }
+ }
+ )

openpype/hosts/webpublisher/plugins/publish/collect_fps.py

@@ -20,9 +20,8 @@ class CollectFPS(pyblish.api.InstancePlugin):
  hosts = ["webpublisher"]
 
  def process(self, instance):
- fps = instance.context.data["fps"]
+ instance_fps = instance.data.get("fps")
+ if instance_fps is None:
+ instance.data["fps"] = instance.context.data["fps"]
 
- instance.data.update({
- "fps": fps
- })
  self.log.debug(f"instance.data: {pformat(instance.data)}")

openpype/hosts/webpublisher/plugins/publish/collect_published_files.py

@@ -1,21 +1,19 @@
-"""Loads publishing context from json and continues in publish process.
+"""Create instances from batch data and continues in publish process.
 
 Requires:
- anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11)
+ CollectBatchData
 
 Provides:
  context, instances -> All data from previous publishing process.
 """
 
 import os
-import json
 import clique
 import tempfile
-
-import pyblish.api
 from avalon import io
+import pyblish.api
 from openpype.lib import prepare_template_data
-from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info
+from openpype.lib.plugin_tools import parse_json
 
 
 class CollectPublishedFiles(pyblish.api.ContextPlugin):
@@ -28,28 +26,28 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin):
  order = pyblish.api.CollectorOrder - 0.490
  label = "Collect rendered frames"
  host = ["webpublisher"]
-
- _context = None
+ targets = ["filespublish"]
 
  # from Settings
  task_type_to_family = {}
 
- def _process_batch(self, dir_url):
- task_subfolders = [
- os.path.join(dir_url, o)
- for o in os.listdir(dir_url)
- if os.path.isdir(os.path.join(dir_url, o))]
+ def process(self, context):
+ batch_dir = context.data["batchDir"]
+ task_subfolders = []
+ for folder_name in os.listdir(batch_dir):
+ full_path = os.path.join(batch_dir, folder_name)
+ if os.path.isdir(full_path):
+ task_subfolders.append(full_path)
+
  self.log.info("task_sub:: {}".format(task_subfolders))
+
+ asset_name = context.data["asset"]
+ task_name = context.data["task"]
+ task_type = context.data["taskType"]
  for task_dir in task_subfolders:
  task_data = parse_json(os.path.join(task_dir,
  "manifest.json"))
  self.log.info("task_data:: {}".format(task_data))
- ctx = task_data["context"]
-
- asset, task_name, task_type = get_batch_asset_task_info(ctx)
-
- if task_name:
- os.environ["AVALON_TASK"] = task_name
 
  is_sequence = len(task_data["files"]) > 1
 
@@ -60,26 +58,20 @@ def _process_batch(self, dir_url):
  is_sequence,
  extension.replace(".", ''))
 
- subset = self._get_subset_name(family, subset_template, task_name,
- task_data["variant"])
-
- os.environ["AVALON_ASSET"] = asset
- io.Session["AVALON_ASSET"] = asset
+ subset = self._get_subset_name(
+ family, subset_template, task_name, task_data["variant"]
+ )
+ version = self._get_last_version(asset_name, subset) + 1
 
- instance = self._context.create_instance(subset)
- instance.data["asset"] = asset
+ instance = context.create_instance(subset)
+ instance.data["asset"] = asset_name
  instance.data["subset"] = subset
  instance.data["family"] = family
  instance.data["families"] = families
- instance.data["version"] = \
- self._get_last_version(asset, subset) + 1
+ instance.data["version"] = version
  instance.data["stagingDir"] = tempfile.mkdtemp()
  instance.data["source"] = "webpublisher"
 
- # to store logging info into DB openpype.webpublishes
- instance.data["ctx_path"] = ctx["path"]
- instance.data["batch_id"] = task_data["batch"]
-
  # to convert from email provided into Ftrack username
  instance.data["user_email"] = task_data["user"]
 
@@ -230,23 +222,3 @@ def _get_last_version(self, asset_name, subset_name):
  return version[0].get("version") or 0
  else:
  return 0
-
- def process(self, context):
- self._context = context
-
- batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
-
- assert batch_dir, (
- "Missing `OPENPYPE_PUBLISH_DATA`")
-
- assert os.path.exists(batch_dir), \
- "Folder {} doesn't exist".format(batch_dir)
-
- project_name = os.environ.get("AVALON_PROJECT")
- if project_name is None:
- raise AssertionError(
- "Environment `AVALON_PROJECT` was not found."
- "Could not set project `root` which may cause issues."
- )
-
- self._process_batch(batch_dir)