From e4c4e1072c28b50d55185df9f85b67cba9937d76 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 22 Jul 2021 21:29:40 +0200 Subject: [PATCH 01/57] Webpublisher backend - added new command for webserver WIP webserver_cli --- openpype/cli.py | 9 + openpype/modules/webserver/webserver_cli.py | 217 ++++++++++++++++++ .../modules/webserver/webserver_module.py | 6 +- openpype/pype_commands.py | 7 + 4 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 openpype/modules/webserver/webserver_cli.py diff --git a/openpype/cli.py b/openpype/cli.py index ec5b04c4683..1065152adb6 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -94,6 +94,15 @@ def eventserver(debug, ) +@main.command() +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +def webpublisherwebserver(debug): + if debug: + os.environ['OPENPYPE_DEBUG'] = "3" + + PypeCommands().launch_webpublisher_webservercli() + + @main.command() @click.argument("output_json_path") @click.option("--project", help="Project name", default=None) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py new file mode 100644 index 00000000000..f3f2fc73d10 --- /dev/null +++ b/openpype/modules/webserver/webserver_cli.py @@ -0,0 +1,217 @@ +import attr +import time +import json +import datetime +from bson.objectid import ObjectId +import collections +from aiohttp.web_response import Response + +from avalon.api import AvalonMongoDB +from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint + +from openpype.api import get_hierarchy + + +class WebpublisherProjectsEndpoint(_RestApiEndpoint): + async def get(self) -> Response: + output = [] + for project_name in self.dbcon.database.collection_names(): + project_doc = self.dbcon.database[project_name].find_one({ + "type": "project" + }) + if project_doc: + ret_val = { + "id": project_doc["_id"], + "name": project_doc["name"] + } + output.append(ret_val) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +@attr.s +class AssetItem(object): + """Data class for Render Layer metadata.""" + id = attr.ib() + name = attr.ib() + + # Render Products + children = attr.ib(init=False, default=attr.Factory(list)) + + +class WebpublisherHiearchyEndpoint(_RestApiEndpoint): + async def get(self, project_name) -> Response: + output = [] + query_projection = { + "_id": 1, + "data.tasks": 1, + "data.visualParent": 1, + "name": 1, + "type": 1, + } + + asset_docs = self.dbcon.database[project_name].find( + {"type": "asset"}, + query_projection + ) + asset_docs_by_id = { + asset_doc["_id"]: asset_doc + for asset_doc in asset_docs + } + + asset_ids = list(asset_docs_by_id.keys()) + result = [] + if asset_ids: + result = self.dbcon.database[project_name].aggregate([ + { + "$match": { + "type": "subset", + "parent": {"$in": asset_ids} + } + }, + { + "$group": { + "_id": "$parent", + "count": {"$sum": 1} + } + } + ]) + + asset_docs_by_parent_id = collections.defaultdict(list) + for asset_doc in asset_docs_by_id.values(): + parent_id = asset_doc["data"].get("visualParent") + asset_docs_by_parent_id[parent_id].append(asset_doc) + + appending_queue = collections.deque() + appending_queue.append((None, "root")) + + asset_items_by_id = {} + non_modifiable_items = set() + assets = {} + + # # # while appending_queue: + # # assets = self._recur_hiearchy(asset_docs_by_parent_id, + # # appending_queue, + # # assets, None) + # while asset_docs_by_parent_id: + # for parent_id, asset_docs in asset_items_by_id.items(): + # asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + + while appending_queue: + parent_id, parent_item_name = appending_queue.popleft() + + asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + + asset_item = assets.get(parent_id) + if not asset_item: + asset_item = AssetItem(str(parent_id), parent_item_name) + + for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): + child_item = AssetItem(str(asset_doc["_id"]), + asset_doc["name"]) + asset_item.children.append(child_item) + if not asset_doc["data"]["tasks"]: + appending_queue.append((asset_doc["_id"], + child_item.name)) + + else: + asset_item = child_item + for task_name, _ in asset_doc["data"]["tasks"].items(): + child_item = AssetItem(str(asset_doc["_id"]), + task_name) + asset_item.children.append(child_item) + assets[parent_id] = attr.asdict(asset_item) + + + return Response( + status=200, + body=self.resource.encode(assets), + content_type="application/json" + ) + + def _recur_hiearchy(self, asset_docs_by_parent_id, + appending_queue, assets, asset_item): + parent_id, parent_item_name = appending_queue.popleft() + + asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + + if not asset_item: + asset_item = assets.get(parent_id) + if not asset_item: + asset_item = AssetItem(str(parent_id), parent_item_name) + + for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): + child_item = AssetItem(str(asset_doc["_id"]), + asset_doc["name"]) + asset_item.children.append(child_item) + if not asset_doc["data"]["tasks"]: + appending_queue.append((asset_doc["_id"], + child_item.name)) + asset_item = child_item + assets = self._recur_hiearchy(asset_docs_by_parent_id, appending_queue, + assets, asset_item) + else: + asset_item = child_item + for task_name, _ in asset_doc["data"]["tasks"].items(): + child_item = AssetItem(str(asset_doc["_id"]), + task_name) + asset_item.children.append(child_item) + assets[asset_item.id] = attr.asdict(asset_item) + + return assets + +class RestApiResource: + def __init__(self, server_manager): + self.server_manager = server_manager + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + @staticmethod + def json_dump_handler(value): + print("valuetype:: {}".format(type(value))) + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, ObjectId): + return str(value) + raise TypeError(value) + + @classmethod + def encode(cls, data): + return json.dumps( + data, + indent=4, + default=cls.json_dump_handler + ).encode("utf-8") + + +def run_webserver(): + print("webserver") + from openpype.modules import ModulesManager + + manager = ModulesManager() + webserver_module = manager.modules_by_name["webserver"] + webserver_module.create_server_manager() + + resource = RestApiResource(webserver_module.server_manager) + projects_endpoint = WebpublisherProjectsEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/webpublisher/projects", + projects_endpoint.dispatch + ) + + hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/webpublisher/hiearchy/{project_name}", + hiearchy_endpoint.dispatch + ) + + webserver_module.start_server() + while True: + time.sleep(0.5) + diff --git a/openpype/modules/webserver/webserver_module.py b/openpype/modules/webserver/webserver_module.py index b61619acded..4832038575a 100644 --- a/openpype/modules/webserver/webserver_module.py +++ b/openpype/modules/webserver/webserver_module.py @@ -50,10 +50,8 @@ def _add_resources_statics(self): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) - webserver_url = "http://localhost:{}".format(self.port) - os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( - webserver_url, static_prefix + os.environ["OPENPYPE_WEBSERVER_URL"], static_prefix ) def _add_listeners(self): @@ -81,6 +79,8 @@ def create_server_manager(self): self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) + webserver_url = "http://localhost:{}".format(self.port) + os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url @staticmethod def find_free_port( diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 7c47d8c6139..6ccf10e8ced 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -40,6 +40,13 @@ def launch_eventservercli(*args): ) return run_event_server(*args) + @staticmethod + def launch_webpublisher_webservercli(*args): + from openpype.modules.webserver.webserver_cli import ( + run_webserver + ) + return run_webserver(*args) + @staticmethod def launch_standalone_publisher(): from openpype.tools import standalonepublish From e4cc3033057c4e33a9a535efd79eb7c74d196f12 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Sun, 25 Jul 2021 15:50:47 +0200 Subject: [PATCH 02/57] Webpublisher backend - implemented context endopoint --- openpype/modules/webserver/webserver_cli.py | 163 ++++++++------------ 1 file changed, 64 insertions(+), 99 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index f3f2fc73d10..3ebbc863585 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -13,6 +13,7 @@ class WebpublisherProjectsEndpoint(_RestApiEndpoint): + """Returns list of project names.""" async def get(self) -> Response: output = [] for project_name in self.dbcon.database.collection_names(): @@ -32,23 +33,44 @@ async def get(self) -> Response: ) -@attr.s -class AssetItem(object): - """Data class for Render Layer metadata.""" - id = attr.ib() - name = attr.ib() +class Node(dict): + """Node element in context tree.""" - # Render Products - children = attr.ib(init=False, default=attr.Factory(list)) + def __init__(self, uid, node_type, name): + self._parent = None # pointer to parent Node + self["type"] = node_type + self["name"] = name + self['id'] = uid # keep reference to id # + self['children'] = [] # collection of pointers to child Nodes + + @property + def parent(self): + return self._parent # simply return the object at the _parent pointer + + @parent.setter + def parent(self, node): + self._parent = node + # add this node to parent's list of children + node['children'].append(self) + + +class TaskNode(Node): + """Special node type only for Tasks.""" + def __init__(self, node_type, name): + self._parent = None + self["type"] = node_type + self["name"] = name + self["attributes"] = {} class WebpublisherHiearchyEndpoint(_RestApiEndpoint): + """Returns dictionary with context tree from assets.""" async def get(self, project_name) -> Response: - output = [] query_projection = { "_id": 1, "data.tasks": 1, "data.visualParent": 1, + "data.entityType": 1, "name": 1, "type": 1, } @@ -62,106 +84,51 @@ async def get(self, project_name) -> Response: for asset_doc in asset_docs } - asset_ids = list(asset_docs_by_id.keys()) - result = [] - if asset_ids: - result = self.dbcon.database[project_name].aggregate([ - { - "$match": { - "type": "subset", - "parent": {"$in": asset_ids} - } - }, - { - "$group": { - "_id": "$parent", - "count": {"$sum": 1} - } - } - ]) - asset_docs_by_parent_id = collections.defaultdict(list) for asset_doc in asset_docs_by_id.values(): parent_id = asset_doc["data"].get("visualParent") asset_docs_by_parent_id[parent_id].append(asset_doc) - appending_queue = collections.deque() - appending_queue.append((None, "root")) - - asset_items_by_id = {} - non_modifiable_items = set() - assets = {} - - # # # while appending_queue: - # # assets = self._recur_hiearchy(asset_docs_by_parent_id, - # # appending_queue, - # # assets, None) - # while asset_docs_by_parent_id: - # for parent_id, asset_docs in asset_items_by_id.items(): - # asset_docs = asset_docs_by_parent_id.get(parent_id) or [] - - while appending_queue: - parent_id, parent_item_name = appending_queue.popleft() - - asset_docs = asset_docs_by_parent_id.get(parent_id) or [] - - asset_item = assets.get(parent_id) - if not asset_item: - asset_item = AssetItem(str(parent_id), parent_item_name) - - for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): - child_item = AssetItem(str(asset_doc["_id"]), - asset_doc["name"]) - asset_item.children.append(child_item) - if not asset_doc["data"]["tasks"]: - appending_queue.append((asset_doc["_id"], - child_item.name)) - - else: - asset_item = child_item - for task_name, _ in asset_doc["data"]["tasks"].items(): - child_item = AssetItem(str(asset_doc["_id"]), - task_name) - asset_item.children.append(child_item) - assets[parent_id] = attr.asdict(asset_item) - + assets = collections.defaultdict(list) + + for parent_id, children in asset_docs_by_parent_id.items(): + for child in children: + node = assets.get(child["_id"]) + if not node: + node = Node(child["_id"], + child["data"]["entityType"], + child["name"]) + assets[child["_id"]] = node + + tasks = child["data"].get("tasks", {}) + for t_name, t_con in tasks.items(): + task_node = TaskNode("task", t_name) + task_node["attributes"]["type"] = t_con.get("type") + + task_node.parent = node + + parent_node = assets.get(parent_id) + if not parent_node: + asset_doc = asset_docs_by_id.get(parent_id) + if asset_doc: # regular node + parent_node = Node(parent_id, + asset_doc["data"]["entityType"], + asset_doc["name"]) + else: # root + parent_node = Node(parent_id, + "project", + project_name) + assets[parent_id] = parent_node + node.parent = parent_node + + roots = [x for x in assets.values() if x.parent is None] return Response( status=200, - body=self.resource.encode(assets), + body=self.resource.encode(roots[0]), content_type="application/json" ) - def _recur_hiearchy(self, asset_docs_by_parent_id, - appending_queue, assets, asset_item): - parent_id, parent_item_name = appending_queue.popleft() - - asset_docs = asset_docs_by_parent_id.get(parent_id) or [] - - if not asset_item: - asset_item = assets.get(parent_id) - if not asset_item: - asset_item = AssetItem(str(parent_id), parent_item_name) - - for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): - child_item = AssetItem(str(asset_doc["_id"]), - asset_doc["name"]) - asset_item.children.append(child_item) - if not asset_doc["data"]["tasks"]: - appending_queue.append((asset_doc["_id"], - child_item.name)) - asset_item = child_item - assets = self._recur_hiearchy(asset_docs_by_parent_id, appending_queue, - assets, asset_item) - else: - asset_item = child_item - for task_name, _ in asset_doc["data"]["tasks"].items(): - child_item = AssetItem(str(asset_doc["_id"]), - task_name) - asset_item.children.append(child_item) - assets[asset_item.id] = attr.asdict(asset_item) - - return assets class RestApiResource: def __init__(self, server_manager): @@ -172,7 +139,6 @@ def __init__(self, server_manager): @staticmethod def json_dump_handler(value): - print("valuetype:: {}".format(type(value))) if isinstance(value, datetime.datetime): return value.isoformat() if isinstance(value, ObjectId): @@ -189,7 +155,6 @@ def encode(cls, data): def run_webserver(): - print("webserver") from openpype.modules import ModulesManager manager = ModulesManager() From 6c32a8e6a36d11e1988933be3ed2d1c4a7c2e51e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 26 Jul 2021 10:15:23 +0200 Subject: [PATCH 03/57] Webpublisher backend - changed uri to api --- openpype/modules/webserver/webserver_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 3ebbc863585..17b98cc1af4 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -165,14 +165,14 @@ def run_webserver(): projects_endpoint = WebpublisherProjectsEndpoint(resource) webserver_module.server_manager.add_route( "GET", - "/webpublisher/projects", + "/api/projects", projects_endpoint.dispatch ) hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) webserver_module.server_manager.add_route( "GET", - "/webpublisher/hiearchy/{project_name}", + "/api/hiearchy/{project_name}", hiearchy_endpoint.dispatch ) From 622ff2a797bcf9b5954f5b0f80ad06482576521a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 26 Jul 2021 11:25:52 +0200 Subject: [PATCH 04/57] Webpublisher backend - changed uri to api --- openpype/modules/webserver/webserver_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 17b98cc1af4..b6317a56750 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -172,7 +172,7 @@ def run_webserver(): hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) webserver_module.server_manager.add_route( "GET", - "/api/hiearchy/{project_name}", + "/api/hierarchy/{project_name}", hiearchy_endpoint.dispatch ) From 32a82b50f4386d6756c24bdf17f3e02f606dd5f1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jul 2021 18:47:15 +0200 Subject: [PATCH 05/57] Webpublisher - backend - added webpublisher host --- openpype/hosts/webpublisher/README.md | 6 + openpype/hosts/webpublisher/__init__.py | 0 .../plugins/collect_published_files.py | 159 ++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 openpype/hosts/webpublisher/README.md create mode 100644 openpype/hosts/webpublisher/__init__.py create mode 100644 openpype/hosts/webpublisher/plugins/collect_published_files.py diff --git a/openpype/hosts/webpublisher/README.md b/openpype/hosts/webpublisher/README.md new file mode 100644 index 00000000000..0826e444902 --- /dev/null +++ b/openpype/hosts/webpublisher/README.md @@ -0,0 +1,6 @@ +Webpublisher +------------- + +Plugins meant for processing of Webpublisher. + +Gets triggered by calling openpype.cli.remotepublish with appropriate arguments. \ No newline at end of file diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/openpype/hosts/webpublisher/plugins/collect_published_files.py b/openpype/hosts/webpublisher/plugins/collect_published_files.py new file mode 100644 index 00000000000..1cc0dfe83fb --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/collect_published_files.py @@ -0,0 +1,159 @@ +"""Loads publishing context from json and continues in publish process. + +Requires: + anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + +Provides: + context, instances -> All data from previous publishing process. +""" + +import os +import json + +import pyblish.api +from avalon import api + + +class CollectPublishedFiles(pyblish.api.ContextPlugin): + """ + This collector will try to find json files in provided + `OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context. + + """ + # must be really early, context values are only in json file + order = pyblish.api.CollectorOrder - 0.495 + label = "Collect rendered frames" + host = ["webpublisher"] + + _context = None + + def _load_json(self, path): + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + self.log.error( + "Error loading json: " + "{} - Exception: {}".format(path, exc) + ) + return data + + def _fill_staging_dir(self, data_object, anatomy): + staging_dir = data_object.get("stagingDir") + if staging_dir: + data_object["stagingDir"] = anatomy.fill_root(staging_dir) + + def _process_path(self, data, anatomy): + # validate basic necessary data + data_err = "invalid json file - missing data" + required = ["asset", "user", "comment", + "job", "instances", "session", "version"] + assert all(elem in data.keys() for elem in required), data_err + + # set context by first json file + ctx = self._context.data + + ctx["asset"] = ctx.get("asset") or data.get("asset") + ctx["intent"] = ctx.get("intent") or data.get("intent") + ctx["comment"] = ctx.get("comment") or data.get("comment") + ctx["user"] = ctx.get("user") or data.get("user") + ctx["version"] = ctx.get("version") or data.get("version") + + # basic sanity check to see if we are working in same context + # if some other json file has different context, bail out. + ctx_err = "inconsistent contexts in json files - %s" + assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" + assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" + assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" + assert ctx.get("user") == data.get("user"), ctx_err % "user" + assert ctx.get("version") == data.get("version"), ctx_err % "version" + + # ftrack credentials are passed as environment variables by Deadline + # to publish job, but Muster doesn't pass them. + if data.get("ftrack") and not os.environ.get("FTRACK_API_USER"): + ftrack = data.get("ftrack") + os.environ["FTRACK_API_USER"] = ftrack["FTRACK_API_USER"] + os.environ["FTRACK_API_KEY"] = ftrack["FTRACK_API_KEY"] + os.environ["FTRACK_SERVER"] = ftrack["FTRACK_SERVER"] + + # now we can just add instances from json file and we are done + for instance_data in data.get("instances"): + self.log.info(" - processing instance for {}".format( + instance_data.get("subset"))) + instance = self._context.create_instance( + instance_data.get("subset") + ) + self.log.info("Filling stagingDir...") + + self._fill_staging_dir(instance_data, anatomy) + instance.data.update(instance_data) + + # stash render job id for later validation + instance.data["render_job_id"] = data.get("job").get("_id") + + representations = [] + for repre_data in instance_data.get("representations") or []: + self._fill_staging_dir(repre_data, anatomy) + representations.append(repre_data) + + instance.data["representations"] = representations + + # add audio if in metadata data + if data.get("audio"): + instance.data.update({ + "audio": [{ + "filename": data.get("audio"), + "offset": 0 + }] + }) + self.log.info( + f"Adding audio to instance: {instance.data['audio']}") + + def process(self, context): + self._context = context + + assert os.environ.get("OPENPYPE_PUBLISH_DATA"), ( + "Missing `OPENPYPE_PUBLISH_DATA`") + paths = os.environ["OPENPYPE_PUBLISH_DATA"].split(os.pathsep) + + project_name = os.environ.get("AVALON_PROJECT") + if project_name is None: + raise AssertionError( + "Environment `AVALON_PROJECT` was not found." + "Could not set project `root` which may cause issues." + ) + + # TODO root filling should happen after collect Anatomy + self.log.info("Getting root setting for project \"{}\"".format( + project_name + )) + + anatomy = context.data["anatomy"] + self.log.info("anatomy: {}".format(anatomy.roots)) + try: + session_is_set = False + for path in paths: + path = anatomy.fill_root(path) + data = self._load_json(path) + assert data, "failed to load json file" + if not session_is_set: + session_data = data["session"] + remapped = anatomy.roots_obj.path_remapper( + session_data["AVALON_WORKDIR"] + ) + if remapped: + session_data["AVALON_WORKDIR"] = remapped + + self.log.info("Setting session using data from file") + api.Session.update(session_data) + os.environ.update(session_data) + session_is_set = True + self._process_path(data, anatomy) + except Exception as e: + self.log.error(e, exc_info=True) + raise Exception("Error") from e From ca1ad20506c99b00412091a42a5cfe8ef28af7bd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jul 2021 18:47:58 +0200 Subject: [PATCH 06/57] Webpublisher - backend - added task_finish endpoint Added scaffolding to run publish process --- openpype/cli.py | 27 ++++++++- openpype/lib/applications.py | 20 ++++--- openpype/modules/webserver/webserver_cli.py | 62 +++++++++++++++++++-- openpype/pype_commands.py | 62 ++++++++++++++++++++- 4 files changed, 151 insertions(+), 20 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 1065152adb6..e56a572c9c1 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -96,11 +96,16 @@ def eventserver(debug, @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") -def webpublisherwebserver(debug): +@click.option("-e", "--executable", help="Executable") +@click.option("-u", "--upload_dir", help="Upload dir") +def webpublisherwebserver(debug, executable, upload_dir): if debug: os.environ['OPENPYPE_DEBUG'] = "3" - PypeCommands().launch_webpublisher_webservercli() + PypeCommands().launch_webpublisher_webservercli( + upload_dir=upload_dir, + executable=executable + ) @main.command() @@ -140,6 +145,24 @@ def publish(debug, paths, targets): PypeCommands.publish(list(paths), targets) +@main.command() +@click.argument("paths", nargs=-1) +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("-h", "--host", help="Host") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets module", default=None, + multiple=True) +def remotepublish(debug, project, paths, host, targets=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + if debug: + os.environ['OPENPYPE_DEBUG'] = '3' + PypeCommands.remotepublish(project, list(paths), host, targets=None) + + @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("-p", "--project", required=True, diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index fb86d061507..1d0d5dcbaa6 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -1004,7 +1004,7 @@ def __init__(self, data): def get_app_environments_for_context( - project_name, asset_name, task_name, app_name, env=None + project_name, asset_name, task_name, app_name=None, env=None ): """Prepare environment variables by context. Args: @@ -1033,20 +1033,14 @@ def get_app_environments_for_context( "name": asset_name }) - # Prepare app object which can be obtained only from ApplciationManager - app_manager = ApplicationManager() - app = app_manager.applications[app_name] - # Project's anatomy anatomy = Anatomy(project_name) - data = EnvironmentPrepData({ + prep_dict = { "project_name": project_name, "asset_name": asset_name, "task_name": task_name, - "app": app, - "dbcon": dbcon, "project_doc": project_doc, "asset_doc": asset_doc, @@ -1054,7 +1048,15 @@ def get_app_environments_for_context( "anatomy": anatomy, "env": env - }) + } + + if app_name: + # Prepare app object which can be obtained only from ApplicationManager + app_manager = ApplicationManager() + app = app_manager.applications[app_name] + prep_dict["app"] = app + + data = EnvironmentPrepData(prep_dict) prepare_host_environments(data) prepare_context_environments(data) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index b6317a56750..00caa24d27c 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -1,16 +1,15 @@ -import attr +import os import time import json import datetime from bson.objectid import ObjectId import collections from aiohttp.web_response import Response +import subprocess from avalon.api import AvalonMongoDB from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint -from openpype.api import get_hierarchy - class WebpublisherProjectsEndpoint(_RestApiEndpoint): """Returns list of project names.""" @@ -130,9 +129,51 @@ async def get(self, project_name) -> Response: ) +class WebpublisherTaskFinishEndpoint(_RestApiEndpoint): + """Returns list of project names.""" + async def post(self, request) -> Response: + output = {} + + print(request) + + json_path = os.path.join(self.resource.upload_dir, + "webpublisher.json") # temp - pull from request + + openpype_app = self.resource.executable + args = [ + openpype_app, + 'remotepublish', + json_path + ] + + if not openpype_app or not os.path.exists(openpype_app): + msg = "Non existent OpenPype executable {}".format(openpype_app) + raise RuntimeError(msg) + + add_args = { + "host": "webpublisher", + "project": request.query["project"] + } + + for key, value in add_args.items(): + args.append("--{}".format(key)) + args.append(value) + + print("args:: {}".format(args)) + + exit_code = subprocess.call(args, shell=True) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + class RestApiResource: - def __init__(self, server_manager): + def __init__(self, server_manager, executable, upload_dir): self.server_manager = server_manager + self.upload_dir = upload_dir + self.executable = executable self.dbcon = AvalonMongoDB() self.dbcon.install() @@ -154,14 +195,16 @@ def encode(cls, data): ).encode("utf-8") -def run_webserver(): +def run_webserver(*args, **kwargs): from openpype.modules import ModulesManager manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - resource = RestApiResource(webserver_module.server_manager) + resource = RestApiResource(webserver_module.server_manager, + upload_dir=kwargs["upload_dir"], + executable=kwargs["executable"]) projects_endpoint = WebpublisherProjectsEndpoint(resource) webserver_module.server_manager.add_route( "GET", @@ -176,6 +219,13 @@ def run_webserver(): hiearchy_endpoint.dispatch ) + task_finish_endpoint = WebpublisherTaskFinishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/task_finish", + task_finish_endpoint.dispatch + ) + webserver_module.start_server() while True: time.sleep(0.5) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 6ccf10e8ced..d2726fd2a64 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -41,11 +41,11 @@ def launch_eventservercli(*args): return run_event_server(*args) @staticmethod - def launch_webpublisher_webservercli(*args): + def launch_webpublisher_webservercli(*args, **kwargs): from openpype.modules.webserver.webserver_cli import ( run_webserver ) - return run_webserver(*args) + return run_webserver(*args, **kwargs) @staticmethod def launch_standalone_publisher(): @@ -53,7 +53,7 @@ def launch_standalone_publisher(): standalonepublish.main() @staticmethod - def publish(paths, targets=None): + def publish(paths, targets=None, host=None): """Start headless publishing. Publish use json from passed paths argument. @@ -111,6 +111,62 @@ def publish(paths, targets=None): log.info("Publish finished.") uninstall() + @staticmethod + def remotepublish(project, paths, host, targets=None): + """Start headless publishing. + + Publish use json from passed paths argument. + + Args: + paths (list): Paths to jsons. + targets (string): What module should be targeted + (to choose validator for example) + host (string) + + Raises: + RuntimeError: When there is no path to process. + """ + if not any(paths): + raise RuntimeError("No publish paths specified") + + from openpype import install, uninstall + from openpype.api import Logger + + # Register target and host + import pyblish.api + import pyblish.util + + log = Logger.get_logger() + + install() + + if host: + pyblish.api.register_host(host) + + if targets: + if isinstance(targets, str): + targets = [targets] + for target in targets: + pyblish.api.register_target(target) + + os.environ["OPENPYPE_PUBLISH_DATA"] = os.pathsep.join(paths) + os.environ["AVALON_PROJECT"] = project + os.environ["AVALON_APP"] = host # to trigger proper plugings + + log.info("Running publish ...") + + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + + for result in pyblish.util.publish_iter(): + if result["error"]: + log.error(error_format.format(**result)) + uninstall() + sys.exit(1) + + log.info("Publish finished.") + uninstall() + def extractenvironments(output_json_path, project, asset, task, app): env = os.environ.copy() if all((project, asset, task, app)): From 52c6bdc0e5669d729402dee9221d3d1a44087109 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:01:09 +0200 Subject: [PATCH 07/57] Webpublisher - backend - skip version collect for webpublisher --- openpype/plugins/publish/collect_scene_version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_scene_version.py b/openpype/plugins/publish/collect_scene_version.py index 669e6752f36..62969858c5b 100644 --- a/openpype/plugins/publish/collect_scene_version.py +++ b/openpype/plugins/publish/collect_scene_version.py @@ -16,7 +16,8 @@ def process(self, context): if "standalonepublisher" in context.data.get("host", []): return - if "unreal" in pyblish.api.registered_hosts(): + if "unreal" in pyblish.api.registered_hosts() or \ + "webpublisher" in pyblish.api.registered_hosts(): return assert context.data.get('currentFile'), "Cannot get current file" From f104d601319efcec086d6a3a44a11c37ec74832e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:03:02 +0200 Subject: [PATCH 08/57] Webpublisher - backend - updated command Added logging to DB for reports --- openpype/cli.py | 7 +++-- openpype/pype_commands.py | 64 +++++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index e56a572c9c1..8dc32b307a6 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -146,13 +146,14 @@ def publish(debug, paths, targets): @main.command() -@click.argument("paths", nargs=-1) +@click.argument("path") @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("-h", "--host", help="Host") +@click.option("-u", "--user", help="User email address") @click.option("-p", "--project", help="Project") @click.option("-t", "--targets", help="Targets module", default=None, multiple=True) -def remotepublish(debug, project, paths, host, targets=None): +def remotepublish(debug, project, path, host, targets=None, user=None): """Start CLI publishing. Publish collects json from paths provided as an argument. @@ -160,7 +161,7 @@ def remotepublish(debug, project, paths, host, targets=None): """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' - PypeCommands.remotepublish(project, list(paths), host, targets=None) + PypeCommands.remotepublish(project, path, host, user, targets=targets) @main.command() diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d2726fd2a64..24becd24239 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -3,7 +3,7 @@ import os import sys import json -from pathlib import Path +from datetime import datetime from openpype.lib import PypeLogger from openpype.api import get_app_environments_for_context @@ -112,25 +112,30 @@ def publish(paths, targets=None, host=None): uninstall() @staticmethod - def remotepublish(project, paths, host, targets=None): + def remotepublish(project, batch_path, host, user, targets=None): """Start headless publishing. Publish use json from passed paths argument. Args: - paths (list): Paths to jsons. + project (str): project to publish (only single context is expected + per call of remotepublish + batch_path (str): Path batch folder. Contains subfolders with + resources (workfile, another subfolder 'renders' etc.) targets (string): What module should be targeted (to choose validator for example) host (string) + user (string): email address for webpublisher Raises: RuntimeError: When there is no path to process. """ - if not any(paths): + if not batch_path: raise RuntimeError("No publish paths specified") from openpype import install, uninstall from openpype.api import Logger + from openpype.lib import OpenPypeMongoConnection # Register target and host import pyblish.api @@ -149,20 +154,67 @@ def remotepublish(project, paths, host, targets=None): for target in targets: pyblish.api.register_target(target) - os.environ["OPENPYPE_PUBLISH_DATA"] = os.pathsep.join(paths) + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host # to trigger proper plugings + os.environ["AVALON_APP_NAME"] = host # to trigger proper plugings + + # this should be more generic + from openpype.hosts.webpublisher.api import install as w_install + w_install() + pyblish.api.register_host(host) log.info("Running publish ...") # Error exit as soon as any error occurs. error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + dbcon = mongo_client[database_name]["webpublishes"] + + _, batch_id = os.path.split(batch_path) + _id = dbcon.insert_one({ + "batch_id": batch_id, + "start_date": datetime.now(), + "user": user, + "status": "in_progress" + }).inserted_id + for result in pyblish.util.publish_iter(): if result["error"]: log.error(error_format.format(**result)) uninstall() + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": "error", + "msg": error_format.format(**result) + } + } + ) sys.exit(1) + else: + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "progress": result["progress"] + } + } + ) + + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "state": "finished_ok", + "progress": 1 + } + } + ) log.info("Publish finished.") uninstall() From a43837ca91983d7251d1bbb8232b302abc29c950 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:03:39 +0200 Subject: [PATCH 09/57] Webpublisher - backend - added collector plugin --- openpype/hosts/webpublisher/api/__init__.py | 36 +++ .../plugins/collect_published_files.py | 159 ---------- .../publish/collect_published_files.py | 292 ++++++++++++++++++ 3 files changed, 328 insertions(+), 159 deletions(-) create mode 100644 openpype/hosts/webpublisher/api/__init__.py delete mode 100644 openpype/hosts/webpublisher/plugins/collect_published_files.py create mode 100644 openpype/hosts/webpublisher/plugins/publish/collect_published_files.py diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py new file mode 100644 index 00000000000..908c9b10be4 --- /dev/null +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -0,0 +1,36 @@ +import os +import logging + +from avalon import api as avalon +from pyblish import api as pyblish +import openpype.hosts.webpublisher + +log = logging.getLogger("openpype.hosts.webpublisher") + +HOST_DIR = os.path.dirname(os.path.abspath( + openpype.hosts.webpublisher.__file__)) +PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") +PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") +LOAD_PATH = os.path.join(PLUGINS_DIR, "load") +CREATE_PATH = os.path.join(PLUGINS_DIR, "create") + + +def application_launch(): + pass + + +def install(): + print("Installing Pype config...") + + pyblish.register_plugin_path(PUBLISH_PATH) + avalon.register_plugin_path(avalon.Loader, LOAD_PATH) + avalon.register_plugin_path(avalon.Creator, CREATE_PATH) + log.info(PUBLISH_PATH) + + avalon.on("application.launched", application_launch) + +def uninstall(): + pyblish.deregister_plugin_path(PUBLISH_PATH) + avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH) + avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH) + diff --git a/openpype/hosts/webpublisher/plugins/collect_published_files.py b/openpype/hosts/webpublisher/plugins/collect_published_files.py deleted file mode 100644 index 1cc0dfe83fb..00000000000 --- a/openpype/hosts/webpublisher/plugins/collect_published_files.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Loads publishing context from json and continues in publish process. - -Requires: - anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) - -Provides: - context, instances -> All data from previous publishing process. -""" - -import os -import json - -import pyblish.api -from avalon import api - - -class CollectPublishedFiles(pyblish.api.ContextPlugin): - """ - This collector will try to find json files in provided - `OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context. - - """ - # must be really early, context values are only in json file - order = pyblish.api.CollectorOrder - 0.495 - label = "Collect rendered frames" - host = ["webpublisher"] - - _context = None - - def _load_json(self, path): - path = path.strip('\"') - assert os.path.isfile(path), ( - "Path to json file doesn't exist. \"{}\"".format(path) - ) - data = None - with open(path, "r") as json_file: - try: - data = json.load(json_file) - except Exception as exc: - self.log.error( - "Error loading json: " - "{} - Exception: {}".format(path, exc) - ) - return data - - def _fill_staging_dir(self, data_object, anatomy): - staging_dir = data_object.get("stagingDir") - if staging_dir: - data_object["stagingDir"] = anatomy.fill_root(staging_dir) - - def _process_path(self, data, anatomy): - # validate basic necessary data - data_err = "invalid json file - missing data" - required = ["asset", "user", "comment", - "job", "instances", "session", "version"] - assert all(elem in data.keys() for elem in required), data_err - - # set context by first json file - ctx = self._context.data - - ctx["asset"] = ctx.get("asset") or data.get("asset") - ctx["intent"] = ctx.get("intent") or data.get("intent") - ctx["comment"] = ctx.get("comment") or data.get("comment") - ctx["user"] = ctx.get("user") or data.get("user") - ctx["version"] = ctx.get("version") or data.get("version") - - # basic sanity check to see if we are working in same context - # if some other json file has different context, bail out. - ctx_err = "inconsistent contexts in json files - %s" - assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" - assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" - assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" - assert ctx.get("user") == data.get("user"), ctx_err % "user" - assert ctx.get("version") == data.get("version"), ctx_err % "version" - - # ftrack credentials are passed as environment variables by Deadline - # to publish job, but Muster doesn't pass them. - if data.get("ftrack") and not os.environ.get("FTRACK_API_USER"): - ftrack = data.get("ftrack") - os.environ["FTRACK_API_USER"] = ftrack["FTRACK_API_USER"] - os.environ["FTRACK_API_KEY"] = ftrack["FTRACK_API_KEY"] - os.environ["FTRACK_SERVER"] = ftrack["FTRACK_SERVER"] - - # now we can just add instances from json file and we are done - for instance_data in data.get("instances"): - self.log.info(" - processing instance for {}".format( - instance_data.get("subset"))) - instance = self._context.create_instance( - instance_data.get("subset") - ) - self.log.info("Filling stagingDir...") - - self._fill_staging_dir(instance_data, anatomy) - instance.data.update(instance_data) - - # stash render job id for later validation - instance.data["render_job_id"] = data.get("job").get("_id") - - representations = [] - for repre_data in instance_data.get("representations") or []: - self._fill_staging_dir(repre_data, anatomy) - representations.append(repre_data) - - instance.data["representations"] = representations - - # add audio if in metadata data - if data.get("audio"): - instance.data.update({ - "audio": [{ - "filename": data.get("audio"), - "offset": 0 - }] - }) - self.log.info( - f"Adding audio to instance: {instance.data['audio']}") - - def process(self, context): - self._context = context - - assert os.environ.get("OPENPYPE_PUBLISH_DATA"), ( - "Missing `OPENPYPE_PUBLISH_DATA`") - paths = os.environ["OPENPYPE_PUBLISH_DATA"].split(os.pathsep) - - project_name = os.environ.get("AVALON_PROJECT") - if project_name is None: - raise AssertionError( - "Environment `AVALON_PROJECT` was not found." - "Could not set project `root` which may cause issues." - ) - - # TODO root filling should happen after collect Anatomy - self.log.info("Getting root setting for project \"{}\"".format( - project_name - )) - - anatomy = context.data["anatomy"] - self.log.info("anatomy: {}".format(anatomy.roots)) - try: - session_is_set = False - for path in paths: - path = anatomy.fill_root(path) - data = self._load_json(path) - assert data, "failed to load json file" - if not session_is_set: - session_data = data["session"] - remapped = anatomy.roots_obj.path_remapper( - session_data["AVALON_WORKDIR"] - ) - if remapped: - session_data["AVALON_WORKDIR"] = remapped - - self.log.info("Setting session using data from file") - api.Session.update(session_data) - os.environ.update(session_data) - session_is_set = True - self._process_path(data, anatomy) - except Exception as e: - self.log.error(e, exc_info=True) - raise Exception("Error") from e diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py new file mode 100644 index 00000000000..69d30e06e15 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -0,0 +1,292 @@ +"""Loads publishing context from json and continues in publish process. + +Requires: + anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + +Provides: + context, instances -> All data from previous publishing process. +""" + +import os +import json +import clique + +import pyblish.api +from avalon import api + +FAMILY_SETTING = { # TEMP + "Animation": { + "workfile": { + "is_sequence": False, + "extensions": ["tvp"], + "families": [] + }, + "render": { + "is_sequence": True, + "extensions": [ + "png", "exr", "tiff", "tif" + ], + "families": ["review"] + } + }, + "Compositing": { + "workfile": { + "is_sequence": False, + "extensions": ["aep"], + "families": [] + }, + "render": { + "is_sequence": True, + "extensions": [ + "png", "exr", "tiff", "tif" + ], + "families": ["review"] + } + }, + "Layout": { + "workfile": { + "is_sequence": False, + "extensions": [ + ".psd" + ], + "families": [] + }, + "image": { + "is_sequence": False, + "extensions": [ + "png", + "jpg", + "jpeg", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + } +} + +class CollectPublishedFiles(pyblish.api.ContextPlugin): + """ + This collector will try to find json files in provided + `OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context. + + """ + # must be really early, context values are only in json file + order = pyblish.api.CollectorOrder - 0.490 + label = "Collect rendered frames" + host = ["webpublisher"] + + _context = None + + def _load_json(self, path): + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + self.log.error( + "Error loading json: " + "{} - Exception: {}".format(path, exc) + ) + return data + + def _fill_staging_dir(self, data_object, anatomy): + staging_dir = data_object.get("stagingDir") + if staging_dir: + data_object["stagingDir"] = anatomy.fill_root(staging_dir) + + def _process_path(self, data): + # validate basic necessary data + data_err = "invalid json file - missing data" + # required = ["asset", "user", "comment", + # "job", "instances", "session", "version"] + # assert all(elem in data.keys() for elem in required), data_err + + # set context by first json file + ctx = self._context.data + + ctx["asset"] = ctx.get("asset") or data.get("asset") + ctx["intent"] = ctx.get("intent") or data.get("intent") + ctx["comment"] = ctx.get("comment") or data.get("comment") + ctx["user"] = ctx.get("user") or data.get("user") + ctx["version"] = ctx.get("version") or data.get("version") + + # basic sanity check to see if we are working in same context + # if some other json file has different context, bail out. + ctx_err = "inconsistent contexts in json files - %s" + assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" + assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" + assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" + assert ctx.get("user") == data.get("user"), ctx_err % "user" + assert ctx.get("version") == data.get("version"), ctx_err % "version" + + # now we can just add instances from json file and we are done + for instance_data in data.get("instances"): + self.log.info(" - processing instance for {}".format( + instance_data.get("subset"))) + instance = self._context.create_instance( + instance_data.get("subset") + ) + self.log.info("Filling stagingDir...") + + self._fill_staging_dir(instance_data, anatomy) + instance.data.update(instance_data) + + # stash render job id for later validation + instance.data["render_job_id"] = data.get("job").get("_id") + + representations = [] + for repre_data in instance_data.get("representations") or []: + self._fill_staging_dir(repre_data, anatomy) + representations.append(repre_data) + + instance.data["representations"] = representations + + # add audio if in metadata data + if data.get("audio"): + instance.data.update({ + "audio": [{ + "filename": data.get("audio"), + "offset": 0 + }] + }) + self.log.info( + f"Adding audio to instance: {instance.data['audio']}") + + def _process_batch(self, dir_url): + task_subfolders = [os.path.join(dir_url, o) + for o in os.listdir(dir_url) + if os.path.isdir(os.path.join(dir_url, o))] + self.log.info("task_sub:: {}".format(task_subfolders)) + for task_dir in task_subfolders: + task_data = self._load_json(os.path.join(task_dir, + "manifest.json")) + self.log.info("task_data:: {}".format(task_data)) + ctx = task_data["context"] + asset = subset = task = task_type = None + + subset = "Main" # temp + if ctx["type"] == "task": + items = ctx["path"].split('/') + asset = items[-2] + os.environ["AVALON_TASK"] = ctx["name"] + task_type = ctx["attributes"]["type"] + else: + asset = ctx["name"] + + is_sequence = len(task_data["files"]) > 1 + + instance = self._context.create_instance(subset) + _, extension = os.path.splitext(task_data["files"][0]) + self.log.info("asset:: {}".format(asset)) + family, families = self._get_family(FAMILY_SETTING, # todo + task_type, + is_sequence, + extension.replace(".", '')) + os.environ["AVALON_ASSET"] = asset + instance.data["asset"] = asset + instance.data["subset"] = subset + instance.data["family"] = family + instance.data["families"] = families + # instance.data["version"] = self._get_version(task_data["subset"]) + instance.data["stagingDir"] = task_dir + instance.data["source"] = "webpublisher" + + os.environ["FTRACK_API_USER"] = task_data["user"] + + if is_sequence: + instance.data["representations"] = self._process_sequence( + task_data["files"], task_dir + ) + else: + _, ext = os.path.splittext(task_data["files"][0]) + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": task_data["files"], + "stagingDir": task_dir + } + instance.data["representation"] = repre_data + + self.log.info("instance.data:: {}".format(instance.data)) + + def _process_sequence(self, files, task_dir): + """Prepare reprentations for sequence of files.""" + collections, remainder = clique.assemble(files) + assert len(collections) == 1, \ + "Too many collections in {}".format(files) + + frame_start = list(collections[0].indexes)[0] + frame_end = list(collections[0].indexes)[-1] + ext = collections[0].tail + repre_data = { + "frameStart": frame_start, + "frameEnd": frame_end, + "name": ext[1:], + "ext": ext[1:], + "files": files, + "stagingDir": task_dir + } + self.log.info("repre_data.data:: {}".format(repre_data)) + return [repre_data] + + def _get_family(self, settings, task_type, is_sequence, extension): + """Guess family based on input data. + + Args: + settings (dict): configuration per task_type + task_type (str): Animation|Art etc + is_sequence (bool): single file or sequence + extension (str): without '.' + + Returns: + (family, [families]) tuple + AssertionError if not matching family found + """ + task_obj = settings.get(task_type) + assert task_obj, "No family configuration for '{}'".format(task_type) + + found_family = None + for family, content in task_obj.items(): + if is_sequence != content["is_sequence"]: + continue + if extension in content["extensions"]: + found_family = family + break + + msg = "No family found for combination of " +\ + "task_type: {}, is_sequence:{}, extension: {}".format( + task_type, is_sequence, extension) + assert found_family, msg + + return found_family, content["families"] + + def _get_version(self, subset_name): + return 1 + + def process(self, context): + self._context = context + + batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") + + assert batch_dir, ( + "Missing `OPENPYPE_PUBLISH_DATA`") + + assert batch_dir, \ + "Folder {} doesn't exist".format(batch_dir) + + project_name = os.environ.get("AVALON_PROJECT") + if project_name is None: + raise AssertionError( + "Environment `AVALON_PROJECT` was not found." + "Could not set project `root` which may cause issues." + ) + + self._process_batch(batch_dir) + From 824714c2f898a6eb37569d88b875a3137c16d1f9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:04:20 +0200 Subject: [PATCH 10/57] Webpublisher - backend - added endpoints for reporting --- openpype/modules/webserver/webserver_cli.py | 126 ++++++++++++++------ 1 file changed, 88 insertions(+), 38 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 00caa24d27c..04d0002787f 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -8,6 +8,8 @@ import subprocess from avalon.api import AvalonMongoDB + +from openpype.lib import OpenPypeMongoConnection from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint @@ -32,36 +34,6 @@ async def get(self) -> Response: ) -class Node(dict): - """Node element in context tree.""" - - def __init__(self, uid, node_type, name): - self._parent = None # pointer to parent Node - self["type"] = node_type - self["name"] = name - self['id'] = uid # keep reference to id # - self['children'] = [] # collection of pointers to child Nodes - - @property - def parent(self): - return self._parent # simply return the object at the _parent pointer - - @parent.setter - def parent(self, node): - self._parent = node - # add this node to parent's list of children - node['children'].append(self) - - -class TaskNode(Node): - """Special node type only for Tasks.""" - def __init__(self, node_type, name): - self._parent = None - self["type"] = node_type - self["name"] = name - self["attributes"] = {} - - class WebpublisherHiearchyEndpoint(_RestApiEndpoint): """Returns dictionary with context tree from assets.""" async def get(self, project_name) -> Response: @@ -129,21 +101,52 @@ async def get(self, project_name) -> Response: ) -class WebpublisherTaskFinishEndpoint(_RestApiEndpoint): +class Node(dict): + """Node element in context tree.""" + + def __init__(self, uid, node_type, name): + self._parent = None # pointer to parent Node + self["type"] = node_type + self["name"] = name + self['id'] = uid # keep reference to id # + self['children'] = [] # collection of pointers to child Nodes + + @property + def parent(self): + return self._parent # simply return the object at the _parent pointer + + @parent.setter + def parent(self, node): + self._parent = node + # add this node to parent's list of children + node['children'].append(self) + + +class TaskNode(Node): + """Special node type only for Tasks.""" + + def __init__(self, node_type, name): + self._parent = None + self["type"] = node_type + self["name"] = name + self["attributes"] = {} + + +class WebpublisherPublishEndpoint(_RestApiEndpoint): """Returns list of project names.""" async def post(self, request) -> Response: output = {} print(request) - json_path = os.path.join(self.resource.upload_dir, - "webpublisher.json") # temp - pull from request + batch_path = os.path.join(self.resource.upload_dir, + request.query["batch_id"]) openpype_app = self.resource.executable args = [ openpype_app, 'remotepublish', - json_path + batch_path ] if not openpype_app or not os.path.exists(openpype_app): @@ -152,7 +155,8 @@ async def post(self, request) -> Response: add_args = { "host": "webpublisher", - "project": request.query["project"] + "project": request.query["project"], + "user": request.query["user"] } for key, value in add_args.items(): @@ -169,6 +173,30 @@ async def post(self, request) -> Response: ) +class BatchStatusEndpoint(_RestApiEndpoint): + """Returns list of project names.""" + async def get(self, batch_id) -> Response: + output = self.dbcon.find_one({"batch_id": batch_id}) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class PublishesStatusEndpoint(_RestApiEndpoint): + """Returns list of project names.""" + async def get(self, user) -> Response: + output = self.dbcon.find({"user": user}) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + class RestApiResource: def __init__(self, server_manager, executable, upload_dir): self.server_manager = server_manager @@ -195,6 +223,13 @@ def encode(cls, data): ).encode("utf-8") +class OpenPypeRestApiResource(RestApiResource): + def __init__(self, ): + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + self.dbcon = mongo_client[database_name]["webpublishes"] + + def run_webserver(*args, **kwargs): from openpype.modules import ModulesManager @@ -219,11 +254,26 @@ def run_webserver(*args, **kwargs): hiearchy_endpoint.dispatch ) - task_finish_endpoint = WebpublisherTaskFinishEndpoint(resource) + webpublisher_publish_endpoint = WebpublisherPublishEndpoint(resource) webserver_module.server_manager.add_route( "POST", - "/api/task_finish", - task_finish_endpoint.dispatch + "/api/webpublish/{batch_id}", + webpublisher_publish_endpoint.dispatch + ) + + openpype_resource = OpenPypeRestApiResource() + batch_status_endpoint = BatchStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/batch_status/{batch_id}", + batch_status_endpoint.dispatch + ) + + user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/publishes/{user}", + user_status_endpoint.dispatch ) webserver_module.start_server() From bfd2ad65cf2877f245192ba8d28548ae1edc0ad2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:10:09 +0200 Subject: [PATCH 11/57] Webpublisher - backend - hound --- openpype/hosts/webpublisher/api/__init__.py | 2 +- .../plugins/publish/collect_published_files.py | 3 +-- openpype/modules/webserver/webserver_cli.py | 3 +-- openpype/pype_commands.py | 9 +++------ 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 908c9b10be4..1b6edcf24d6 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -29,8 +29,8 @@ def install(): avalon.on("application.launched", application_launch) + def uninstall(): pyblish.deregister_plugin_path(PUBLISH_PATH) avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH) avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH) - diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 69d30e06e15..dde9713c7a2 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -169,7 +169,7 @@ def _process_batch(self, dir_url): "manifest.json")) self.log.info("task_data:: {}".format(task_data)) ctx = task_data["context"] - asset = subset = task = task_type = None + task_type = None subset = "Main" # temp if ctx["type"] == "task": @@ -289,4 +289,3 @@ def process(self, context): ) self._process_batch(batch_dir) - diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 04d0002787f..484c25c6b35 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -165,7 +165,7 @@ async def post(self, request) -> Response: print("args:: {}".format(args)) - exit_code = subprocess.call(args, shell=True) + _exit_code = subprocess.call(args, shell=True) return Response( status=200, body=self.resource.encode(output), @@ -279,4 +279,3 @@ def run_webserver(*args, **kwargs): webserver_module.start_server() while True: time.sleep(0.5) - diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 24becd24239..01fa6b8d333 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -191,8 +191,7 @@ def remotepublish(project, batch_path, host, user, targets=None): "finish_date": datetime.now(), "status": "error", "msg": error_format.format(**result) - } - } + }} ) sys.exit(1) else: @@ -201,8 +200,7 @@ def remotepublish(project, batch_path, host, user, targets=None): {"$set": { "progress": result["progress"] - } - } + }} ) dbcon.update_one( @@ -212,8 +210,7 @@ def remotepublish(project, batch_path, host, user, targets=None): "finish_date": datetime.now(), "state": "finished_ok", "progress": 1 - } - } + }} ) log.info("Publish finished.") From 7decf0aa911a3fd18d7a91688ae39fd6f098eb13 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Jul 2021 11:06:17 +0200 Subject: [PATCH 12/57] Webpublisher - backend - added settings and defaults --- .../project_settings/webpublisher.json | 72 +++++++++++++++++++ .../schemas/projects_schema/schema_main.json | 4 ++ .../schema_project_webpublisher.json | 60 ++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 openpype/settings/defaults/project_settings/webpublisher.json create mode 100644 openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json new file mode 100644 index 00000000000..69b6babc648 --- /dev/null +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -0,0 +1,72 @@ +{ + "publish": { + "CollectPublishedFiles": { + "task_type_to_family": { + "Animation": { + "workfile": { + "is_sequence": false, + "extensions": [ + "tvp" + ], + "families": [] + }, + "render": { + "is_sequence": true, + "extensions": [ + "png", + "exr", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + }, + "Compositing": { + "workfile": { + "is_sequence": false, + "extensions": [ + "aep" + ], + "families": [] + }, + "render": { + "is_sequence": true, + "extensions": [ + "png", + "exr", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + }, + "Layout": { + "workfile": { + "is_sequence": false, + "extensions": [ + "psd" + ], + "families": [] + }, + "image": { + "is_sequence": false, + "extensions": [ + "png", + "jpg", + "jpeg", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + } + } + } + } +} \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_main.json b/openpype/settings/entities/schemas/projects_schema/schema_main.json index 4a8a9d496ea..575cfc9e727 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_main.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_main.json @@ -118,6 +118,10 @@ "type": "schema", "name": "schema_project_standalonepublisher" }, + { + "type": "schema", + "name": "schema_project_webpublisher" + }, { "type": "schema", "name": "schema_project_unreal" diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json new file mode 100644 index 00000000000..6ae82e05619 --- /dev/null +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json @@ -0,0 +1,60 @@ +{ + "type": "dict", + "collapsible": true, + "key": "webpublisher", + "label": "Web Publisher", + "is_file": true, + "children": [ + { + "type": "dict", + "collapsible": true, + "key": "publish", + "label": "Publish plugins", + "children": [ + { + "type": "dict", + "collapsible": true, + "key": "CollectPublishedFiles", + "label": "Collect Published Files", + "children": [ + { + "type": "dict-modifiable", + "collapsible": true, + "key": "task_type_to_family", + "label": "Task type to family mapping", + "collapsible_key": true, + "object_type": { + "type": "dict-modifiable", + "collapsible": false, + "key": "task_type", + "collapsible_key": false, + "object_type": { + "type": "dict", + "children": [ + { + "type": "boolean", + "key": "is_sequence", + "label": "Is Sequence" + }, + { + "type": "list", + "key": "extensions", + "label": "Extensions", + "object_type": "text" + }, + { + "type": "list", + "key": "families", + "label": "Families", + "object_type": "text" + } + ] + } + } + } + ] + } + ] + } + ] +} \ No newline at end of file From 4f63e3d21ffd0e62caef178f8acb5fe2e422f8c3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Jul 2021 17:30:34 +0200 Subject: [PATCH 13/57] Webpublisher - backend - updated settings --- .../project_settings/webpublisher.json | 44 ++++++++++++++++--- .../schema_project_webpublisher.json | 5 +++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json index 69b6babc648..8364b6a39de 100644 --- a/openpype/settings/defaults/project_settings/webpublisher.json +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -8,7 +8,8 @@ "extensions": [ "tvp" ], - "families": [] + "families": [], + "subset_template_name": "" }, "render": { "is_sequence": true, @@ -20,7 +21,8 @@ ], "families": [ "review" - ] + ], + "subset_template_name": "" } }, "Compositing": { @@ -29,7 +31,8 @@ "extensions": [ "aep" ], - "families": [] + "families": [], + "subset_template_name": "" }, "render": { "is_sequence": true, @@ -41,7 +44,8 @@ ], "families": [ "review" - ] + ], + "subset_template_name": "" } }, "Layout": { @@ -50,7 +54,8 @@ "extensions": [ "psd" ], - "families": [] + "families": [], + "subset_template_name": "" }, "image": { "is_sequence": false, @@ -63,8 +68,35 @@ ], "families": [ "review" - ] + ], + "subset_template_name": "" } + }, + "default_task_type": { + "workfile": { + "is_sequence": false, + "extensions": [ + "tvp" + ], + "families": [], + "subset_template_name": "{family}{Variant}" + }, + "render": { + "is_sequence": true, + "extensions": [ + "png", + "exr", + "tiff", + "tif" + ], + "families": [ + "review" + ], + "subset_template_name": "{family}{Variant}" + } + }, + "__dynamic_keys_labels__": { + "default_task_type": "Default task type" } } } diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json index 6ae82e05619..bf59cd030e9 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json @@ -47,6 +47,11 @@ "key": "families", "label": "Families", "object_type": "text" + }, + { + "type": "text", + "key": "subset_template_name", + "label": "Subset template name" } ] } From dea843d851162624e10a810d431e5ed78c1e13cb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Jul 2021 18:17:19 +0200 Subject: [PATCH 14/57] Webpublisher - backend - implemented version and subset name --- openpype/hosts/webpublisher/api/__init__.py | 2 + .../publish/collect_published_files.py | 237 ++++++++---------- openpype/modules/webserver/webserver_cli.py | 13 +- openpype/pype_commands.py | 3 +- 4 files changed, 112 insertions(+), 143 deletions(-) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 1b6edcf24d6..76709bb2d74 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -2,6 +2,7 @@ import logging from avalon import api as avalon +from avalon import io from pyblish import api as pyblish import openpype.hosts.webpublisher @@ -27,6 +28,7 @@ def install(): avalon.register_plugin_path(avalon.Creator, CREATE_PATH) log.info(PUBLISH_PATH) + io.install() avalon.on("application.launched", application_launch) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index dde9713c7a2..deadbb856b0 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -12,60 +12,9 @@ import clique import pyblish.api -from avalon import api - -FAMILY_SETTING = { # TEMP - "Animation": { - "workfile": { - "is_sequence": False, - "extensions": ["tvp"], - "families": [] - }, - "render": { - "is_sequence": True, - "extensions": [ - "png", "exr", "tiff", "tif" - ], - "families": ["review"] - } - }, - "Compositing": { - "workfile": { - "is_sequence": False, - "extensions": ["aep"], - "families": [] - }, - "render": { - "is_sequence": True, - "extensions": [ - "png", "exr", "tiff", "tif" - ], - "families": ["review"] - } - }, - "Layout": { - "workfile": { - "is_sequence": False, - "extensions": [ - ".psd" - ], - "families": [] - }, - "image": { - "is_sequence": False, - "extensions": [ - "png", - "jpg", - "jpeg", - "tiff", - "tif" - ], - "families": [ - "review" - ] - } - } -} +from avalon import io +from openpype.lib import prepare_template_data + class CollectPublishedFiles(pyblish.api.ContextPlugin): """ @@ -80,6 +29,9 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): _context = None + # from Settings + task_type_to_family = {} + def _load_json(self, path): path = path.strip('\"') assert os.path.isfile(path), ( @@ -96,69 +48,6 @@ def _load_json(self, path): ) return data - def _fill_staging_dir(self, data_object, anatomy): - staging_dir = data_object.get("stagingDir") - if staging_dir: - data_object["stagingDir"] = anatomy.fill_root(staging_dir) - - def _process_path(self, data): - # validate basic necessary data - data_err = "invalid json file - missing data" - # required = ["asset", "user", "comment", - # "job", "instances", "session", "version"] - # assert all(elem in data.keys() for elem in required), data_err - - # set context by first json file - ctx = self._context.data - - ctx["asset"] = ctx.get("asset") or data.get("asset") - ctx["intent"] = ctx.get("intent") or data.get("intent") - ctx["comment"] = ctx.get("comment") or data.get("comment") - ctx["user"] = ctx.get("user") or data.get("user") - ctx["version"] = ctx.get("version") or data.get("version") - - # basic sanity check to see if we are working in same context - # if some other json file has different context, bail out. - ctx_err = "inconsistent contexts in json files - %s" - assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" - assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" - assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" - assert ctx.get("user") == data.get("user"), ctx_err % "user" - assert ctx.get("version") == data.get("version"), ctx_err % "version" - - # now we can just add instances from json file and we are done - for instance_data in data.get("instances"): - self.log.info(" - processing instance for {}".format( - instance_data.get("subset"))) - instance = self._context.create_instance( - instance_data.get("subset") - ) - self.log.info("Filling stagingDir...") - - self._fill_staging_dir(instance_data, anatomy) - instance.data.update(instance_data) - - # stash render job id for later validation - instance.data["render_job_id"] = data.get("job").get("_id") - - representations = [] - for repre_data in instance_data.get("representations") or []: - self._fill_staging_dir(repre_data, anatomy) - representations.append(repre_data) - - instance.data["representations"] = representations - - # add audio if in metadata data - if data.get("audio"): - instance.data.update({ - "audio": [{ - "filename": data.get("audio"), - "offset": 0 - }] - }) - self.log.info( - f"Adding audio to instance: {instance.data['audio']}") - def _process_batch(self, dir_url): task_subfolders = [os.path.join(dir_url, o) for o in os.listdir(dir_url) @@ -169,32 +58,41 @@ def _process_batch(self, dir_url): "manifest.json")) self.log.info("task_data:: {}".format(task_data)) ctx = task_data["context"] - task_type = None + task_type = "default_task_type" + task_name = None subset = "Main" # temp if ctx["type"] == "task": items = ctx["path"].split('/') asset = items[-2] os.environ["AVALON_TASK"] = ctx["name"] + task_name = ctx["name"] task_type = ctx["attributes"]["type"] else: asset = ctx["name"] is_sequence = len(task_data["files"]) > 1 - instance = self._context.create_instance(subset) _, extension = os.path.splitext(task_data["files"][0]) self.log.info("asset:: {}".format(asset)) - family, families = self._get_family(FAMILY_SETTING, # todo - task_type, - is_sequence, - extension.replace(".", '')) + family, families, subset_template = self._get_family( + self.task_type_to_family, + task_type, + is_sequence, + extension.replace(".", '')) + + subset = self._get_subset_name(family, subset_template, task_name, + task_data["variant"]) + os.environ["AVALON_ASSET"] = asset + io.Session["AVALON_ASSET"] = asset + + instance = self._context.create_instance(subset) instance.data["asset"] = asset instance.data["subset"] = subset instance.data["family"] = family instance.data["families"] = families - # instance.data["version"] = self._get_version(task_data["subset"]) + instance.data["version"] = self._get_version(asset, subset) + 1 instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" @@ -205,17 +103,33 @@ def _process_batch(self, dir_url): task_data["files"], task_dir ) else: - _, ext = os.path.splittext(task_data["files"][0]) - repre_data = { - "name": ext[1:], - "ext": ext[1:], - "files": task_data["files"], - "stagingDir": task_dir - } - instance.data["representation"] = repre_data + + instance.data["representation"] = self._get_single_repre( + task_dir, task_data["files"] + ) self.log.info("instance.data:: {}".format(instance.data)) + def _get_subset_name(self, family, subset_template, task_name, variant): + fill_pairs = { + "variant": variant, + "family": family, + "task": task_name + } + subset = subset_template.format(**prepare_template_data(fill_pairs)) + return subset + + def _get_single_repre(self, task_dir, files): + _, ext = os.path.splittext(files[0]) + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": files, + "stagingDir": task_dir + } + + return repre_data + def _process_sequence(self, files, task_dir): """Prepare reprentations for sequence of files.""" collections, remainder = clique.assemble(files) @@ -246,7 +160,7 @@ def _get_family(self, settings, task_type, is_sequence, extension): extension (str): without '.' Returns: - (family, [families]) tuple + (family, [families], subset_template_name) tuple AssertionError if not matching family found """ task_obj = settings.get(task_type) @@ -265,10 +179,59 @@ def _get_family(self, settings, task_type, is_sequence, extension): task_type, is_sequence, extension) assert found_family, msg - return found_family, content["families"] - - def _get_version(self, subset_name): - return 1 + return found_family, \ + content["families"], \ + content["subset_template_name"] + + def _get_version(self, asset_name, subset_name): + """Returns version number or 0 for 'asset' and 'subset'""" + query = [ + { + "$match": {"type": "asset", "name": asset_name} + }, + { + "$lookup": + { + "from": os.environ["AVALON_PROJECT"], + "localField": "_id", + "foreignField": "parent", + "as": "subsets" + } + }, + { + "$unwind": "$subsets" + }, + { + "$match": {"subsets.type": "subset", + "subsets.name": subset_name}}, + { + "$lookup": + { + "from": os.environ["AVALON_PROJECT"], + "localField": "subsets._id", + "foreignField": "parent", + "as": "versions" + } + }, + { + "$unwind": "$versions" + }, + { + "$group": { + "_id": { + "asset_name": "$name", + "subset_name": "$subsets.name" + }, + 'version': {'$max': "$versions.name"} + } + } + ] + version = list(io.aggregate(query)) + + if version: + return version[0].get("version") or 0 + else: + return 0 def process(self, context): self._context = context diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 484c25c6b35..7773bde5677 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -146,7 +146,8 @@ async def post(self, request) -> Response: args = [ openpype_app, 'remotepublish', - batch_path + batch_id, + task_id ] if not openpype_app or not os.path.exists(openpype_app): @@ -174,7 +175,7 @@ async def post(self, request) -> Response: class BatchStatusEndpoint(_RestApiEndpoint): - """Returns list of project names.""" + """Returns dict with info for batch_id.""" async def get(self, batch_id) -> Response: output = self.dbcon.find_one({"batch_id": batch_id}) @@ -186,9 +187,9 @@ async def get(self, batch_id) -> Response: class PublishesStatusEndpoint(_RestApiEndpoint): - """Returns list of project names.""" + """Returns list of dict with batch info for user (email address).""" async def get(self, user) -> Response: - output = self.dbcon.find({"user": user}) + output = list(self.dbcon.find({"user": user})) return Response( status=200, @@ -198,6 +199,7 @@ async def get(self, user) -> Response: class RestApiResource: + """Resource carrying needed info and Avalon DB connection for publish.""" def __init__(self, server_manager, executable, upload_dir): self.server_manager = server_manager self.upload_dir = upload_dir @@ -224,6 +226,7 @@ def encode(cls, data): class OpenPypeRestApiResource(RestApiResource): + """Resource carrying OP DB connection for storing batch info into DB.""" def __init__(self, ): mongo_client = OpenPypeMongoConnection.get_mongo_client() database_name = os.environ["OPENPYPE_DATABASE_NAME"] @@ -254,6 +257,7 @@ def run_webserver(*args, **kwargs): hiearchy_endpoint.dispatch ) + # triggers publish webpublisher_publish_endpoint = WebpublisherPublishEndpoint(resource) webserver_module.server_manager.add_route( "POST", @@ -261,6 +265,7 @@ def run_webserver(*args, **kwargs): webpublisher_publish_endpoint.dispatch ) + # reporting openpype_resource = OpenPypeRestApiResource() batch_status_endpoint = BatchStatusEndpoint(openpype_resource) webserver_module.server_manager.add_route( diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 01fa6b8d333..1391c36661b 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -161,7 +161,6 @@ def remotepublish(project, batch_path, host, user, targets=None): # this should be more generic from openpype.hosts.webpublisher.api import install as w_install w_install() - pyblish.api.register_host(host) log.info("Running publish ...") @@ -199,7 +198,7 @@ def remotepublish(project, batch_path, host, user, targets=None): {"_id": _id}, {"$set": { - "progress": result["progress"] + "progress": max(result["progress"], 0.95) }} ) From 3c7f6a89fe7e72fd808d23c975306a800126579a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 10:22:16 +0200 Subject: [PATCH 15/57] Webpublisher - backend - refactored routes --- .../modules/webserver/webpublish_routes.py | 242 ++++++++++++++++ openpype/modules/webserver/webserver_cli.py | 258 ++---------------- 2 files changed, 265 insertions(+), 235 deletions(-) create mode 100644 openpype/modules/webserver/webpublish_routes.py diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py new file mode 100644 index 00000000000..805ac11a543 --- /dev/null +++ b/openpype/modules/webserver/webpublish_routes.py @@ -0,0 +1,242 @@ +"""Routes and etc. for webpublisher API.""" +import os +import json +import datetime +from bson.objectid import ObjectId +import collections +from aiohttp.web_response import Response +import subprocess + +from avalon.api import AvalonMongoDB + +from openpype.lib import OpenPypeMongoConnection +from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint + + +class RestApiResource: + """Resource carrying needed info and Avalon DB connection for publish.""" + def __init__(self, server_manager, executable, upload_dir): + self.server_manager = server_manager + self.upload_dir = upload_dir + self.executable = executable + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + @staticmethod + def json_dump_handler(value): + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, ObjectId): + return str(value) + raise TypeError(value) + + @classmethod + def encode(cls, data): + return json.dumps( + data, + indent=4, + default=cls.json_dump_handler + ).encode("utf-8") + + +class OpenPypeRestApiResource(RestApiResource): + """Resource carrying OP DB connection for storing batch info into DB.""" + def __init__(self, ): + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + self.dbcon = mongo_client[database_name]["webpublishes"] + + +class WebpublisherProjectsEndpoint(_RestApiEndpoint): + """Returns list of dict with project info (id, name).""" + async def get(self) -> Response: + output = [] + for project_name in self.dbcon.database.collection_names(): + project_doc = self.dbcon.database[project_name].find_one({ + "type": "project" + }) + if project_doc: + ret_val = { + "id": project_doc["_id"], + "name": project_doc["name"] + } + output.append(ret_val) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class WebpublisherHiearchyEndpoint(_RestApiEndpoint): + """Returns dictionary with context tree from assets.""" + async def get(self, project_name) -> Response: + query_projection = { + "_id": 1, + "data.tasks": 1, + "data.visualParent": 1, + "data.entityType": 1, + "name": 1, + "type": 1, + } + + asset_docs = self.dbcon.database[project_name].find( + {"type": "asset"}, + query_projection + ) + asset_docs_by_id = { + asset_doc["_id"]: asset_doc + for asset_doc in asset_docs + } + + asset_docs_by_parent_id = collections.defaultdict(list) + for asset_doc in asset_docs_by_id.values(): + parent_id = asset_doc["data"].get("visualParent") + asset_docs_by_parent_id[parent_id].append(asset_doc) + + assets = collections.defaultdict(list) + + for parent_id, children in asset_docs_by_parent_id.items(): + for child in children: + node = assets.get(child["_id"]) + if not node: + node = Node(child["_id"], + child["data"]["entityType"], + child["name"]) + assets[child["_id"]] = node + + tasks = child["data"].get("tasks", {}) + for t_name, t_con in tasks.items(): + task_node = TaskNode("task", t_name) + task_node["attributes"]["type"] = t_con.get("type") + + task_node.parent = node + + parent_node = assets.get(parent_id) + if not parent_node: + asset_doc = asset_docs_by_id.get(parent_id) + if asset_doc: # regular node + parent_node = Node(parent_id, + asset_doc["data"]["entityType"], + asset_doc["name"]) + else: # root + parent_node = Node(parent_id, + "project", + project_name) + assets[parent_id] = parent_node + node.parent = parent_node + + roots = [x for x in assets.values() if x.parent is None] + + return Response( + status=200, + body=self.resource.encode(roots[0]), + content_type="application/json" + ) + + +class Node(dict): + """Node element in context tree.""" + + def __init__(self, uid, node_type, name): + self._parent = None # pointer to parent Node + self["type"] = node_type + self["name"] = name + self['id'] = uid # keep reference to id # + self['children'] = [] # collection of pointers to child Nodes + + @property + def parent(self): + return self._parent # simply return the object at the _parent pointer + + @parent.setter + def parent(self, node): + self._parent = node + # add this node to parent's list of children + node['children'].append(self) + + +class TaskNode(Node): + """Special node type only for Tasks.""" + + def __init__(self, node_type, name): + self._parent = None + self["type"] = node_type + self["name"] = name + self["attributes"] = {} + + +class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): + """Triggers headless publishing of batch.""" + async def post(self, request) -> Response: + output = {} + + print(request) + + batch_path = os.path.join(self.resource.upload_dir, + request.query["batch_id"]) + + openpype_app = self.resource.executable + args = [ + openpype_app, + 'remotepublish', + batch_path + ] + + if not openpype_app or not os.path.exists(openpype_app): + msg = "Non existent OpenPype executable {}".format(openpype_app) + raise RuntimeError(msg) + + add_args = { + "host": "webpublisher", + "project": request.query["project"], + "user": request.query["user"] + } + + for key, value in add_args.items(): + args.append("--{}".format(key)) + args.append(value) + + print("args:: {}".format(args)) + + _exit_code = subprocess.call(args, shell=True) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class WebpublisherTaskPublishEndpoint(_RestApiEndpoint): + """Prepared endpoint triggered after each task - for future development.""" + async def post(self, request) -> Response: + return Response( + status=200, + body=self.resource.encode([]), + content_type="application/json" + ) + + +class BatchStatusEndpoint(_RestApiEndpoint): + """Returns dict with info for batch_id.""" + async def get(self, batch_id) -> Response: + output = self.dbcon.find_one({"batch_id": batch_id}) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class PublishesStatusEndpoint(_RestApiEndpoint): + """Returns list of dict with batch info for user (email address).""" + async def get(self, user) -> Response: + output = list(self.dbcon.find({"user": user})) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 7773bde5677..0812bfa372a 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -1,239 +1,18 @@ -import os import time -import json -import datetime -from bson.objectid import ObjectId -import collections -from aiohttp.web_response import Response -import subprocess - -from avalon.api import AvalonMongoDB - -from openpype.lib import OpenPypeMongoConnection -from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint - - -class WebpublisherProjectsEndpoint(_RestApiEndpoint): - """Returns list of project names.""" - async def get(self) -> Response: - output = [] - for project_name in self.dbcon.database.collection_names(): - project_doc = self.dbcon.database[project_name].find_one({ - "type": "project" - }) - if project_doc: - ret_val = { - "id": project_doc["_id"], - "name": project_doc["name"] - } - output.append(ret_val) - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class WebpublisherHiearchyEndpoint(_RestApiEndpoint): - """Returns dictionary with context tree from assets.""" - async def get(self, project_name) -> Response: - query_projection = { - "_id": 1, - "data.tasks": 1, - "data.visualParent": 1, - "data.entityType": 1, - "name": 1, - "type": 1, - } - - asset_docs = self.dbcon.database[project_name].find( - {"type": "asset"}, - query_projection - ) - asset_docs_by_id = { - asset_doc["_id"]: asset_doc - for asset_doc in asset_docs - } - - asset_docs_by_parent_id = collections.defaultdict(list) - for asset_doc in asset_docs_by_id.values(): - parent_id = asset_doc["data"].get("visualParent") - asset_docs_by_parent_id[parent_id].append(asset_doc) - - assets = collections.defaultdict(list) - - for parent_id, children in asset_docs_by_parent_id.items(): - for child in children: - node = assets.get(child["_id"]) - if not node: - node = Node(child["_id"], - child["data"]["entityType"], - child["name"]) - assets[child["_id"]] = node - - tasks = child["data"].get("tasks", {}) - for t_name, t_con in tasks.items(): - task_node = TaskNode("task", t_name) - task_node["attributes"]["type"] = t_con.get("type") - - task_node.parent = node - - parent_node = assets.get(parent_id) - if not parent_node: - asset_doc = asset_docs_by_id.get(parent_id) - if asset_doc: # regular node - parent_node = Node(parent_id, - asset_doc["data"]["entityType"], - asset_doc["name"]) - else: # root - parent_node = Node(parent_id, - "project", - project_name) - assets[parent_id] = parent_node - node.parent = parent_node - - roots = [x for x in assets.values() if x.parent is None] - - return Response( - status=200, - body=self.resource.encode(roots[0]), - content_type="application/json" - ) - - -class Node(dict): - """Node element in context tree.""" - - def __init__(self, uid, node_type, name): - self._parent = None # pointer to parent Node - self["type"] = node_type - self["name"] = name - self['id'] = uid # keep reference to id # - self['children'] = [] # collection of pointers to child Nodes - - @property - def parent(self): - return self._parent # simply return the object at the _parent pointer - - @parent.setter - def parent(self, node): - self._parent = node - # add this node to parent's list of children - node['children'].append(self) - - -class TaskNode(Node): - """Special node type only for Tasks.""" - - def __init__(self, node_type, name): - self._parent = None - self["type"] = node_type - self["name"] = name - self["attributes"] = {} - - -class WebpublisherPublishEndpoint(_RestApiEndpoint): - """Returns list of project names.""" - async def post(self, request) -> Response: - output = {} - - print(request) - - batch_path = os.path.join(self.resource.upload_dir, - request.query["batch_id"]) - - openpype_app = self.resource.executable - args = [ - openpype_app, - 'remotepublish', - batch_id, - task_id - ] - - if not openpype_app or not os.path.exists(openpype_app): - msg = "Non existent OpenPype executable {}".format(openpype_app) - raise RuntimeError(msg) - - add_args = { - "host": "webpublisher", - "project": request.query["project"], - "user": request.query["user"] - } - - for key, value in add_args.items(): - args.append("--{}".format(key)) - args.append(value) - - print("args:: {}".format(args)) - - _exit_code = subprocess.call(args, shell=True) - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class BatchStatusEndpoint(_RestApiEndpoint): - """Returns dict with info for batch_id.""" - async def get(self, batch_id) -> Response: - output = self.dbcon.find_one({"batch_id": batch_id}) - - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class PublishesStatusEndpoint(_RestApiEndpoint): - """Returns list of dict with batch info for user (email address).""" - async def get(self, user) -> Response: - output = list(self.dbcon.find({"user": user})) - - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class RestApiResource: - """Resource carrying needed info and Avalon DB connection for publish.""" - def __init__(self, server_manager, executable, upload_dir): - self.server_manager = server_manager - self.upload_dir = upload_dir - self.executable = executable - - self.dbcon = AvalonMongoDB() - self.dbcon.install() - - @staticmethod - def json_dump_handler(value): - if isinstance(value, datetime.datetime): - return value.isoformat() - if isinstance(value, ObjectId): - return str(value) - raise TypeError(value) - - @classmethod - def encode(cls, data): - return json.dumps( - data, - indent=4, - default=cls.json_dump_handler - ).encode("utf-8") - - -class OpenPypeRestApiResource(RestApiResource): - """Resource carrying OP DB connection for storing batch info into DB.""" - def __init__(self, ): - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - self.dbcon = mongo_client[database_name]["webpublishes"] +from .webpublish_routes import ( + RestApiResource, + OpenPypeRestApiResource, + WebpublisherBatchPublishEndpoint, + WebpublisherTaskPublishEndpoint, + WebpublisherHiearchyEndpoint, + WebpublisherProjectsEndpoint, + BatchStatusEndpoint, + PublishesStatusEndpoint +) def run_webserver(*args, **kwargs): + """Runs webserver in command line, adds routes.""" from openpype.modules import ModulesManager manager = ModulesManager() @@ -258,11 +37,20 @@ def run_webserver(*args, **kwargs): ) # triggers publish - webpublisher_publish_endpoint = WebpublisherPublishEndpoint(resource) + webpublisher_task_publish_endpoint = \ + WebpublisherBatchPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/batch", + webpublisher_task_publish_endpoint.dispatch + ) + + webpublisher_batch_publish_endpoint = \ + WebpublisherTaskPublishEndpoint(resource) webserver_module.server_manager.add_route( "POST", - "/api/webpublish/{batch_id}", - webpublisher_publish_endpoint.dispatch + "/api/webpublish/task", + webpublisher_batch_publish_endpoint.dispatch ) # reporting From 349ddf6d915dff324827bf891b71f4a3026841ab Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 10:22:29 +0200 Subject: [PATCH 16/57] Webpublisher - backend - fix signature --- openpype/pype_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 1391c36661b..a4a5cf7a4b9 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -215,6 +215,7 @@ def remotepublish(project, batch_path, host, user, targets=None): log.info("Publish finished.") uninstall() + @staticmethod def extractenvironments(output_json_path, project, asset, task, app): env = os.environ.copy() if all((project, asset, task, app)): From f12df9af7bbccc2487e9e491413c2df0b8b1be77 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 15:44:04 +0200 Subject: [PATCH 17/57] Webpublisher - backend - fix entityType as optional Fix payload for WebpublisherBatchPublishEndpoint --- openpype/modules/webserver/webpublish_routes.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py index 805ac11a543..cf6e4920b68 100644 --- a/openpype/modules/webserver/webpublish_routes.py +++ b/openpype/modules/webserver/webpublish_routes.py @@ -102,7 +102,7 @@ async def get(self, project_name) -> Response: node = assets.get(child["_id"]) if not node: node = Node(child["_id"], - child["data"]["entityType"], + child["data"].get("entityType", "Folder"), child["name"]) assets[child["_id"]] = node @@ -118,7 +118,8 @@ async def get(self, project_name) -> Response: asset_doc = asset_docs_by_id.get(parent_id) if asset_doc: # regular node parent_node = Node(parent_id, - asset_doc["data"]["entityType"], + asset_doc["data"].get("entityType", + "Folder"), asset_doc["name"]) else: # root parent_node = Node(parent_id, @@ -173,9 +174,10 @@ async def post(self, request) -> Response: output = {} print(request) + content = await request.json() batch_path = os.path.join(self.resource.upload_dir, - request.query["batch_id"]) + content["batch"]) openpype_app = self.resource.executable args = [ @@ -190,8 +192,8 @@ async def post(self, request) -> Response: add_args = { "host": "webpublisher", - "project": request.query["project"], - "user": request.query["user"] + "project": content["project_name"], + "user": content["user"] } for key, value in add_args.items(): From 61be1cbb14b82e986bdbb9f650c50b0bd279183d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 15:44:32 +0200 Subject: [PATCH 18/57] Webpublisher - backend - fix app name --- openpype/pype_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index a4a5cf7a4b9..513d7d0865c 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -156,7 +156,7 @@ def remotepublish(project, batch_path, host, user, targets=None): os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP_NAME"] = host # to trigger proper plugings + os.environ["AVALON_APP"] = host # to trigger proper plugings # this should be more generic from openpype.hosts.webpublisher.api import install as w_install From 59ff9225d1a659ea2a84b019cddb93261c887143 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 15:45:15 +0200 Subject: [PATCH 19/57] Webpublisher - backend - set to session for Ftrack family collector --- openpype/hosts/webpublisher/api/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 76709bb2d74..1bf1ef1a6f1 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -29,6 +29,7 @@ def install(): log.info(PUBLISH_PATH) io.install() + avalon.Session["AVALON_APP"] = "webpublisher" # because of Ftrack collect avalon.on("application.launched", application_launch) From 276482e43520dd40fd15e880b86b3eac05fcc310 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 2 Aug 2021 17:16:23 +0200 Subject: [PATCH 20/57] Webpublisher - backend - fixes for single file publish --- .../plugins/publish/collect_published_files.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index deadbb856b0..67d743278be 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -61,7 +61,6 @@ def _process_batch(self, dir_url): task_type = "default_task_type" task_name = None - subset = "Main" # temp if ctx["type"] == "task": items = ctx["path"].split('/') asset = items[-2] @@ -74,7 +73,6 @@ def _process_batch(self, dir_url): is_sequence = len(task_data["files"]) > 1 _, extension = os.path.splitext(task_data["files"][0]) - self.log.info("asset:: {}".format(asset)) family, families, subset_template = self._get_family( self.task_type_to_family, task_type, @@ -103,8 +101,7 @@ def _process_batch(self, dir_url): task_data["files"], task_dir ) else: - - instance.data["representation"] = self._get_single_repre( + instance.data["representations"] = self._get_single_repre( task_dir, task_data["files"] ) @@ -120,15 +117,15 @@ def _get_subset_name(self, family, subset_template, task_name, variant): return subset def _get_single_repre(self, task_dir, files): - _, ext = os.path.splittext(files[0]) + _, ext = os.path.splitext(files[0]) repre_data = { "name": ext[1:], "ext": ext[1:], - "files": files, + "files": files[0], "stagingDir": task_dir } - - return repre_data + self.log.info("single file repre_data.data:: {}".format(repre_data)) + return [repre_data] def _process_sequence(self, files, task_dir): """Prepare reprentations for sequence of files.""" @@ -147,7 +144,7 @@ def _process_sequence(self, files, task_dir): "files": files, "stagingDir": task_dir } - self.log.info("repre_data.data:: {}".format(repre_data)) + self.log.info("sequences repre_data.data:: {}".format(repre_data)) return [repre_data] def _get_family(self, settings, task_type, is_sequence, extension): @@ -170,7 +167,8 @@ def _get_family(self, settings, task_type, is_sequence, extension): for family, content in task_obj.items(): if is_sequence != content["is_sequence"]: continue - if extension in content["extensions"]: + if extension in content["extensions"] or \ + '' in content["extensions"]: # all extensions setting found_family = family break From 63a0c66c881e47c42a1943635a0ed10b72f80a29 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 2 Aug 2021 17:18:07 +0200 Subject: [PATCH 21/57] Webpublisher - backend - fix - removed shell flag causing problems on Linux --- openpype/modules/webserver/webpublish_routes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py index cf6e4920b68..5322802130d 100644 --- a/openpype/modules/webserver/webpublish_routes.py +++ b/openpype/modules/webserver/webpublish_routes.py @@ -12,6 +12,10 @@ from openpype.lib import OpenPypeMongoConnection from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint +from openpype.lib import PypeLogger + +log = PypeLogger.get_logger("WebServer") + class RestApiResource: """Resource carrying needed info and Avalon DB connection for publish.""" @@ -172,8 +176,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): """Triggers headless publishing of batch.""" async def post(self, request) -> Response: output = {} - - print(request) + log.info("WebpublisherBatchPublishEndpoint called") content = await request.json() batch_path = os.path.join(self.resource.upload_dir, @@ -200,9 +203,9 @@ async def post(self, request) -> Response: args.append("--{}".format(key)) args.append(value) - print("args:: {}".format(args)) + log.info("args:: {}".format(args)) - _exit_code = subprocess.call(args, shell=True) + _exit_code = subprocess.call(args) return Response( status=200, body=self.resource.encode(output), From 40f44edd6f5a3f1234995eab51a9d8265d0430aa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 3 Aug 2021 09:53:04 +0200 Subject: [PATCH 22/57] Webpublisher - backend - fix - wrong key in DB --- openpype/pype_commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 513d7d0865c..17b6d58ffd5 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -143,6 +143,8 @@ def remotepublish(project, batch_path, host, user, targets=None): log = Logger.get_logger() + log.info("remotepublish command") + install() if host: @@ -207,7 +209,7 @@ def remotepublish(project, batch_path, host, user, targets=None): {"$set": { "finish_date": datetime.now(), - "state": "finished_ok", + "status": "finished_ok", "progress": 1 }} ) From 8c5941dde81cc520c032cdba901c7fb5611b9dc9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Aug 2021 12:28:50 +0200 Subject: [PATCH 23/57] Webpublisher - added webpublisher host to extract burnin and review --- openpype/plugins/publish/extract_burnin.py | 3 ++- openpype/plugins/publish/extract_review.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index ef52d513259..809cf438c8a 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -44,7 +44,8 @@ class ExtractBurnin(openpype.api.Extractor): "harmony", "fusion", "aftereffects", - "tvpaint" + "tvpaint", + "webpublisher" # "resolve" ] optional = True diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index de54b554e39..07e40b0421d 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -44,7 +44,8 @@ class ExtractReview(pyblish.api.InstancePlugin): "standalonepublisher", "fusion", "tvpaint", - "resolve" + "resolve", + "webpublisher" ] # Supported extensions From 64834df4003c04014dcec7e88fb2b58041ff82b2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Aug 2021 16:43:55 +0200 Subject: [PATCH 24/57] Fix - Deadline publish on Linux started Tray instead of headless publishing --- vendor/deadline/custom/plugins/GlobalJobPreLoad.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/deadline/custom/plugins/GlobalJobPreLoad.py b/vendor/deadline/custom/plugins/GlobalJobPreLoad.py index 41df9d4dc9e..8631b035cf1 100644 --- a/vendor/deadline/custom/plugins/GlobalJobPreLoad.py +++ b/vendor/deadline/custom/plugins/GlobalJobPreLoad.py @@ -55,9 +55,9 @@ def inject_openpype_environment(deadlinePlugin): "AVALON_TASK, AVALON_APP_NAME" raise RuntimeError(msg) - print("args::{}".format(args)) + print("args:::{}".format(args)) - exit_code = subprocess.call(args, shell=True) + exit_code = subprocess.call(args, cwd=os.path.dirname(openpype_app)) if exit_code != 0: raise RuntimeError("Publishing failed, check worker's log") From 96021daebd7cbc7e13108a797e837134bcdc664c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Aug 2021 11:28:06 +0200 Subject: [PATCH 25/57] creating thumbnails from exr in webpublisher --- .../plugins/publish/{extract_jpeg.py => extract_jpeg_exr.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename openpype/plugins/publish/{extract_jpeg.py => extract_jpeg_exr.py} (98%) diff --git a/openpype/plugins/publish/extract_jpeg.py b/openpype/plugins/publish/extract_jpeg_exr.py similarity index 98% rename from openpype/plugins/publish/extract_jpeg.py rename to openpype/plugins/publish/extract_jpeg_exr.py index b1289217e68..8d9e48b6345 100644 --- a/openpype/plugins/publish/extract_jpeg.py +++ b/openpype/plugins/publish/extract_jpeg_exr.py @@ -17,7 +17,7 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin): "imagesequence", "render", "render2d", "source", "plate", "take" ] - hosts = ["shell", "fusion", "resolve"] + hosts = ["shell", "fusion", "resolve", "webpublisher"] enabled = False # presetable attribute From 9c56eb3b53bc76ca68cefe2e1b9ed6975e3d02f1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 11:48:02 +0200 Subject: [PATCH 26/57] Webpublisher - added translation from email to username --- .../plugins/publish/collect_username.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 openpype/hosts/webpublisher/plugins/publish/collect_username.py diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/hosts/webpublisher/plugins/publish/collect_username.py new file mode 100644 index 00000000000..25d6f190a38 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_username.py @@ -0,0 +1,45 @@ +"""Loads publishing context from json and continues in publish process. + +Requires: + anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + +Provides: + context, instances -> All data from previous publishing process. +""" + +import ftrack_api +import os + +import pyblish.api + + +class CollectUsername(pyblish.api.ContextPlugin): + """ + Translates user email to Ftrack username. + + Emails in Ftrack are same as company's Slack, username is needed to + load data to Ftrack. + + """ + order = pyblish.api.CollectorOrder - 0.488 + label = "Collect ftrack username" + host = ["webpublisher"] + + _context = None + + def process(self, context): + os.environ["FTRACK_API_USER"] = "pype.club" + os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] + self.log.info("CollectUsername") + for instance in context: + email = instance.data["user_email"] + self.log.info("email:: {}".format(email)) + session = ftrack_api.Session(auto_connect_event_hub=False) + user = session.query("User where email like '{}'".format( + email)) + + if not user: + raise ValueError("Couldnt find user with {} email".format(email)) + + os.environ["FTRACK_API_USER"] = user[0].get("username") + break From 3d0b470e36f6dee5fc8b5f0160357f73f80254e6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 11:48:19 +0200 Subject: [PATCH 27/57] Webpublisher - added collector for fps --- .../plugins/publish/collect_fps.py | 28 +++++++++++++++++++ .../publish/collect_published_files.py | 13 +++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 openpype/hosts/webpublisher/plugins/publish/collect_fps.py diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_fps.py b/openpype/hosts/webpublisher/plugins/publish/collect_fps.py new file mode 100644 index 00000000000..79fe53176a9 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_fps.py @@ -0,0 +1,28 @@ +""" +Requires: + Nothing + +Provides: + Instance +""" + +import pyblish.api +from pprint import pformat + + +class CollectFPS(pyblish.api.InstancePlugin): + """ + Adds fps from context to instance because of ExtractReview + """ + + label = "Collect fps" + order = pyblish.api.CollectorOrder + 0.49 + hosts = ["webpublisher"] + + def process(self, instance): + fps = instance.context.data["fps"] + + instance.data.update({ + "fps": fps + }) + self.log.debug(f"instance.data: {pformat(instance.data)}") diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 67d743278be..5bc13dff963 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -69,6 +69,7 @@ def _process_batch(self, dir_url): task_type = ctx["attributes"]["type"] else: asset = ctx["name"] + os.environ["AVALON_TASK"] = "" is_sequence = len(task_data["files"]) > 1 @@ -94,12 +95,16 @@ def _process_batch(self, dir_url): instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" - os.environ["FTRACK_API_USER"] = task_data["user"] + instance.data["user_email"] = task_data["user"] if is_sequence: instance.data["representations"] = self._process_sequence( task_data["files"], task_dir ) + instance.data["frameStart"] = \ + instance.data["representations"][0]["frameStart"] + instance.data["frameEnd"] = \ + instance.data["representations"][0]["frameEnd"] else: instance.data["representations"] = self._get_single_repre( task_dir, task_data["files"] @@ -122,7 +127,8 @@ def _get_single_repre(self, task_dir, files): "name": ext[1:], "ext": ext[1:], "files": files[0], - "stagingDir": task_dir + "stagingDir": task_dir, + "tags": ["review"] } self.log.info("single file repre_data.data:: {}".format(repre_data)) return [repre_data] @@ -142,7 +148,8 @@ def _process_sequence(self, files, task_dir): "name": ext[1:], "ext": ext[1:], "files": files, - "stagingDir": task_dir + "stagingDir": task_dir, + "tags": ["review"] } self.log.info("sequences repre_data.data:: {}".format(repre_data)) return [repre_data] From 45fbdcbb564606b59c5f96a8a1232cd2bf596974 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 14:23:45 +0200 Subject: [PATCH 28/57] Webpublisher - added storing full log to Mongo --- openpype/pype_commands.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 17b6d58ffd5..19981d2a39a 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -181,17 +181,26 @@ def remotepublish(project, batch_path, host, user, targets=None): "status": "in_progress" }).inserted_id + log_lines = [] for result in pyblish.util.publish_iter(): + for record in result["records"]: + log_lines.append("{}: {}".format( + result["plugin"].label, record.msg)) + if result["error"]: log.error(error_format.format(**result)) uninstall() + log_lines.append(error_format.format(**result)) dbcon.update_one( {"_id": _id}, {"$set": { "finish_date": datetime.now(), "status": "error", - "msg": error_format.format(**result) + "msg": "Publishing failed > click here and paste " + "report to slack OpenPype support", + "log": os.linesep.join(log_lines) + }} ) sys.exit(1) @@ -200,7 +209,8 @@ def remotepublish(project, batch_path, host, user, targets=None): {"_id": _id}, {"$set": { - "progress": max(result["progress"], 0.95) + "progress": max(result["progress"], 0.95), + "log": os.linesep.join(log_lines) }} ) @@ -210,7 +220,8 @@ def remotepublish(project, batch_path, host, user, targets=None): { "finish_date": datetime.now(), "status": "finished_ok", - "progress": 1 + "progress": 1, + "log": os.linesep.join(log_lines) }} ) From f459791902877c5c6f3e2a13217e2fe52a5bf70d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 16:59:13 +0200 Subject: [PATCH 29/57] Webpublisher - added reprocess functionality Added system settings to enable webpublish --- openpype/modules/webserver/webserver_cli.py | 163 ++++++++++++------ .../defaults/system_settings/modules.json | 3 + .../schemas/system_schema/schema_modules.json | 14 ++ 3 files changed, 127 insertions(+), 53 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 0812bfa372a..dcaa0b4e7bd 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -1,4 +1,10 @@ import time +import os +from datetime import datetime +import requests +import json + + from .webpublish_routes import ( RestApiResource, OpenPypeRestApiResource, @@ -10,6 +16,8 @@ PublishesStatusEndpoint ) +from openpype.api import get_system_settings + def run_webserver(*args, **kwargs): """Runs webserver in command line, adds routes.""" @@ -19,56 +27,105 @@ def run_webserver(*args, **kwargs): webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - resource = RestApiResource(webserver_module.server_manager, - upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"]) - projects_endpoint = WebpublisherProjectsEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/projects", - projects_endpoint.dispatch - ) - - hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/hierarchy/{project_name}", - hiearchy_endpoint.dispatch - ) - - # triggers publish - webpublisher_task_publish_endpoint = \ - WebpublisherBatchPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/batch", - webpublisher_task_publish_endpoint.dispatch - ) - - webpublisher_batch_publish_endpoint = \ - WebpublisherTaskPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/task", - webpublisher_batch_publish_endpoint.dispatch - ) - - # reporting - openpype_resource = OpenPypeRestApiResource() - batch_status_endpoint = BatchStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/batch_status/{batch_id}", - batch_status_endpoint.dispatch - ) - - user_status_endpoint = PublishesStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/publishes/{user}", - user_status_endpoint.dispatch - ) - - webserver_module.start_server() - while True: - time.sleep(0.5) + is_webpublish_enabled = get_system_settings()["modules"]\ + ["webpublish_tool"]["enabled"] + + if is_webpublish_enabled: + resource = RestApiResource(webserver_module.server_manager, + upload_dir=kwargs["upload_dir"], + executable=kwargs["executable"]) + projects_endpoint = WebpublisherProjectsEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/projects", + projects_endpoint.dispatch + ) + + hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/hierarchy/{project_name}", + hiearchy_endpoint.dispatch + ) + + # triggers publish + webpublisher_task_publish_endpoint = \ + WebpublisherBatchPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/batch", + webpublisher_task_publish_endpoint.dispatch + ) + + webpublisher_batch_publish_endpoint = \ + WebpublisherTaskPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/task", + webpublisher_batch_publish_endpoint.dispatch + ) + + # reporting + openpype_resource = OpenPypeRestApiResource() + batch_status_endpoint = BatchStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/batch_status/{batch_id}", + batch_status_endpoint.dispatch + ) + + user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/publishes/{user}", + user_status_endpoint.dispatch + ) + + webserver_module.start_server() + last_reprocessed = time.time() + while True: + if is_webpublish_enabled: + if time.time() - last_reprocessed > 60: + reprocess_failed(kwargs["upload_dir"]) + last_reprocessed = time.time() + time.sleep(1.0) + + +def reprocess_failed(upload_dir): + print("reprocess_failed") + from openpype.lib import OpenPypeMongoConnection + + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + dbcon = mongo_client[database_name]["webpublishes"] + + results = dbcon.find({"status": "reprocess"}) + + for batch in results: + print("batch:: {}".format(batch)) + batch_url = os.path.join(upload_dir, + batch["batch_id"], + "manifest.json") + if not os.path.exists(batch_url): + msg = "Manifest {} not found".format(batch_url) + print(msg) + dbcon.update_one( + {"_id": batch["_id"]}, + {"$set": + { + "finish_date": datetime.now(), + "status": "error", + "progress": 1, + "log": batch.get("log") + msg + }} + ) + continue + + server_url = "{}/api/webpublish/batch".format( + os.environ["OPENPYPE_WEBSERVER_URL"]) + + with open(batch_url) as f: + data = json.loads(f.read()) + + r = requests.post(server_url, json=data) + print(r.status_code) \ No newline at end of file diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 1b74b4695cd..3f9b098a96c 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -165,6 +165,9 @@ "standalonepublish_tool": { "enabled": true }, + "webpublish_tool": { + "enabled": false + }, "project_manager": { "enabled": true }, diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 7d734ff4fd5..f82c3632a97 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -195,6 +195,20 @@ } ] }, + { + "type": "dict", + "key": "webpublish_tool", + "label": "Web Publish", + "collapsible": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + } + ] + }, { "type": "dict", "key": "project_manager", From d2a34a6c712b65de90e206616128a01ddfe82c4e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 12:52:44 +0200 Subject: [PATCH 30/57] Webpublisher - added reprocess functionality --- openpype/modules/webserver/webserver_cli.py | 46 +++++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index dcaa0b4e7bd..2eee20f8557 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -4,6 +4,7 @@ import requests import json +from openpype.lib import PypeLogger from .webpublish_routes import ( RestApiResource, @@ -18,6 +19,10 @@ from openpype.api import get_system_settings +SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost + +log = PypeLogger().get_logger("webserver_gui") + def run_webserver(*args, **kwargs): """Runs webserver in command line, adds routes.""" @@ -27,9 +32,14 @@ def run_webserver(*args, **kwargs): webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - is_webpublish_enabled = get_system_settings()["modules"]\ - ["webpublish_tool"]["enabled"] + is_webpublish_enabled = False + webpublish_tool = get_system_settings()["modules"].\ + get("webpublish_tool") + + if webpublish_tool and webpublish_tool["enabled"]: + is_webpublish_enabled = True + log.debug("is_webpublish_enabled {}".format(is_webpublish_enabled)) if is_webpublish_enabled: resource = RestApiResource(webserver_module.server_manager, upload_dir=kwargs["upload_dir"], @@ -81,18 +91,18 @@ def run_webserver(*args, **kwargs): user_status_endpoint.dispatch ) - webserver_module.start_server() - last_reprocessed = time.time() - while True: - if is_webpublish_enabled: - if time.time() - last_reprocessed > 60: - reprocess_failed(kwargs["upload_dir"]) - last_reprocessed = time.time() - time.sleep(1.0) + webserver_module.start_server() + last_reprocessed = time.time() + while True: + if is_webpublish_enabled: + if time.time() - last_reprocessed > 20: + reprocess_failed(kwargs["upload_dir"]) + last_reprocessed = time.time() + time.sleep(1.0) def reprocess_failed(upload_dir): - print("reprocess_failed") + # log.info("check_reprocesable_records") from openpype.lib import OpenPypeMongoConnection mongo_client = OpenPypeMongoConnection.get_mongo_client() @@ -100,12 +110,11 @@ def reprocess_failed(upload_dir): dbcon = mongo_client[database_name]["webpublishes"] results = dbcon.find({"status": "reprocess"}) - for batch in results: - print("batch:: {}".format(batch)) batch_url = os.path.join(upload_dir, batch["batch_id"], "manifest.json") + log.info("batch:: {} {}".format(os.path.exists(batch_url), batch_url)) if not os.path.exists(batch_url): msg = "Manifest {} not found".format(batch_url) print(msg) @@ -120,12 +129,13 @@ def reprocess_failed(upload_dir): }} ) continue - - server_url = "{}/api/webpublish/batch".format( - os.environ["OPENPYPE_WEBSERVER_URL"]) + server_url = "{}/api/webpublish/batch".format(SERVER_URL) with open(batch_url) as f: data = json.loads(f.read()) - r = requests.post(server_url, json=data) - print(r.status_code) \ No newline at end of file + try: + r = requests.post(server_url, json=data) + log.info("response{}".format(r)) + except: + log.info("exception", exc_info=True) From 932ae5fbb4014e9886a8e679ce8c2b7859439199 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 13:03:25 +0200 Subject: [PATCH 31/57] Hound --- .../plugins/publish/collect_published_files.py | 11 ++++++----- .../webpublisher/plugins/publish/collect_username.py | 6 +++++- openpype/modules/webserver/webpublish_routes.py | 2 +- openpype/pype_commands.py | 2 -- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 5bc13dff963..cd231a0efcf 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -49,9 +49,10 @@ def _load_json(self, path): return data def _process_batch(self, dir_url): - task_subfolders = [os.path.join(dir_url, o) - for o in os.listdir(dir_url) - if os.path.isdir(os.path.join(dir_url, o))] + task_subfolders = [ + os.path.join(dir_url, o) + for o in os.listdir(dir_url) + if os.path.isdir(os.path.join(dir_url, o))] self.log.info("task_sub:: {}".format(task_subfolders)) for task_dir in task_subfolders: task_data = self._load_json(os.path.join(task_dir, @@ -185,8 +186,8 @@ def _get_family(self, settings, task_type, is_sequence, extension): assert found_family, msg return found_family, \ - content["families"], \ - content["subset_template_name"] + content["families"], \ + content["subset_template_name"] def _get_version(self, asset_name, subset_name): """Returns version number or 0 for 'asset' and 'subset'""" diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/hosts/webpublisher/plugins/publish/collect_username.py index 25d6f190a38..0c2c6310f4e 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_username.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_username.py @@ -20,6 +20,9 @@ class CollectUsername(pyblish.api.ContextPlugin): Emails in Ftrack are same as company's Slack, username is needed to load data to Ftrack. + Expects "pype.club" user created on Ftrack and FTRACK_BOT_API_KEY env + var set up. + """ order = pyblish.api.CollectorOrder - 0.488 label = "Collect ftrack username" @@ -39,7 +42,8 @@ def process(self, context): email)) if not user: - raise ValueError("Couldnt find user with {} email".format(email)) + raise ValueError( + "Couldnt find user with {} email".format(email)) os.environ["FTRACK_API_USER"] = user[0].get("username") break diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py index 5322802130d..32feb276ed6 100644 --- a/openpype/modules/webserver/webpublish_routes.py +++ b/openpype/modules/webserver/webpublish_routes.py @@ -205,7 +205,7 @@ async def post(self, request) -> Response: log.info("args:: {}".format(args)) - _exit_code = subprocess.call(args) + subprocess.call(args) return Response( status=200, body=self.resource.encode(output), diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 19981d2a39a..d288e9f2a3e 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -197,8 +197,6 @@ def remotepublish(project, batch_path, host, user, targets=None): { "finish_date": datetime.now(), "status": "error", - "msg": "Publishing failed > click here and paste " - "report to slack OpenPype support", "log": os.linesep.join(log_lines) }} From c6dad89b3478f2c59b60fdcc6ffadbba61d2c1c2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 14:22:42 +0200 Subject: [PATCH 32/57] Webpublisher - added configurable tags + defaults --- .../publish/collect_published_files.py | 19 ++++++++++--------- .../project_settings/webpublisher.json | 16 ++++++++++++++++ .../schema_project_webpublisher.json | 4 ++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index cd231a0efcf..0c89bde8a56 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -75,7 +75,7 @@ def _process_batch(self, dir_url): is_sequence = len(task_data["files"]) > 1 _, extension = os.path.splitext(task_data["files"][0]) - family, families, subset_template = self._get_family( + family, families, subset_template, tags = self._get_family( self.task_type_to_family, task_type, is_sequence, @@ -100,7 +100,7 @@ def _process_batch(self, dir_url): if is_sequence: instance.data["representations"] = self._process_sequence( - task_data["files"], task_dir + task_data["files"], task_dir, tags ) instance.data["frameStart"] = \ instance.data["representations"][0]["frameStart"] @@ -108,7 +108,7 @@ def _process_batch(self, dir_url): instance.data["representations"][0]["frameEnd"] else: instance.data["representations"] = self._get_single_repre( - task_dir, task_data["files"] + task_dir, task_data["files"], tags ) self.log.info("instance.data:: {}".format(instance.data)) @@ -122,19 +122,19 @@ def _get_subset_name(self, family, subset_template, task_name, variant): subset = subset_template.format(**prepare_template_data(fill_pairs)) return subset - def _get_single_repre(self, task_dir, files): + def _get_single_repre(self, task_dir, files, tags): _, ext = os.path.splitext(files[0]) repre_data = { "name": ext[1:], "ext": ext[1:], "files": files[0], "stagingDir": task_dir, - "tags": ["review"] + "tags": tags } self.log.info("single file repre_data.data:: {}".format(repre_data)) return [repre_data] - def _process_sequence(self, files, task_dir): + def _process_sequence(self, files, task_dir, tags): """Prepare reprentations for sequence of files.""" collections, remainder = clique.assemble(files) assert len(collections) == 1, \ @@ -150,7 +150,7 @@ def _process_sequence(self, files, task_dir): "ext": ext[1:], "files": files, "stagingDir": task_dir, - "tags": ["review"] + "tags": tags } self.log.info("sequences repre_data.data:: {}".format(repre_data)) return [repre_data] @@ -165,7 +165,7 @@ def _get_family(self, settings, task_type, is_sequence, extension): extension (str): without '.' Returns: - (family, [families], subset_template_name) tuple + (family, [families], subset_template_name, tags) tuple AssertionError if not matching family found """ task_obj = settings.get(task_type) @@ -187,7 +187,8 @@ def _get_family(self, settings, task_type, is_sequence, extension): return found_family, \ content["families"], \ - content["subset_template_name"] + content["subset_template_name"], \ + content["tags"] def _get_version(self, asset_name, subset_name): """Returns version number or 0 for 'asset' and 'subset'""" diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json index 8364b6a39de..a6916de1447 100644 --- a/openpype/settings/defaults/project_settings/webpublisher.json +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -9,6 +9,7 @@ "tvp" ], "families": [], + "tags": [], "subset_template_name": "" }, "render": { @@ -22,6 +23,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "" } }, @@ -32,6 +36,7 @@ "aep" ], "families": [], + "tags": [], "subset_template_name": "" }, "render": { @@ -45,6 +50,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "" } }, @@ -55,6 +63,7 @@ "psd" ], "families": [], + "tags": [], "subset_template_name": "" }, "image": { @@ -69,6 +78,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "" } }, @@ -79,6 +91,7 @@ "tvp" ], "families": [], + "tags": [], "subset_template_name": "{family}{Variant}" }, "render": { @@ -92,6 +105,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "{family}{Variant}" } }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json index bf59cd030e9..91337da2b27 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json @@ -48,6 +48,10 @@ "label": "Families", "object_type": "text" }, + { + "type": "schema", + "name": "schema_representation_tags" + }, { "type": "text", "key": "subset_template_name", From 4a09a18275ec6b218f62f53a4f5b8e6cfd408d62 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 17:37:33 +0200 Subject: [PATCH 33/57] Webpublisher - fix - status wasn't changed for reprocessed batches --- openpype/modules/webserver/webserver_cli.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 2eee20f8557..8e4dfd229d2 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -139,3 +139,13 @@ def reprocess_failed(upload_dir): log.info("response{}".format(r)) except: log.info("exception", exc_info=True) + + dbcon.update_one( + {"_id": batch["_id"]}, + {"$set": + { + "finish_date": datetime.now(), + "status": "sent_for_reprocessing", + "progress": 1 + }} + ) From 080069ca9c18643276285904124f7508eca44d8a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Aug 2021 16:00:34 +0200 Subject: [PATCH 34/57] Webpublisher - added review to enum, changed defaults This defaults result in creating working review --- .../settings/defaults/project_settings/webpublisher.json | 8 ++++---- .../schemas/schema_representation_tags.json | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json index a6916de1447..f57b79a609d 100644 --- a/openpype/settings/defaults/project_settings/webpublisher.json +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -24,7 +24,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "" } @@ -51,7 +51,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "" } @@ -79,7 +79,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "" } @@ -106,7 +106,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "{family}{Variant}" } diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json index b65de747e54..7607e1a8c14 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json @@ -8,7 +8,10 @@ "burnin": "Add burnins" }, { - "ftrackreview": "Add to Ftrack" + "review": "Create review" + }, + { + "ftrackreview": "Add review to Ftrack" }, { "delete": "Delete output" From 92ef09444695e16427d24b381ba0f513c90b3903 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Aug 2021 17:18:44 +0200 Subject: [PATCH 35/57] Webpublisher - added path field to log documents --- .../publish/collect_published_files.py | 4 ++ .../publish/integrate_context_to_log.py | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 0c89bde8a56..8861190003e 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -96,6 +96,10 @@ def _process_batch(self, dir_url): instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" + instance.data["ctx_path"] = ctx["path"] # to store for logging + instance.data["batch_id"] = task_data["batch"] + + instance.data["user_email"] = task_data["user"] if is_sequence: diff --git a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py new file mode 100644 index 00000000000..1dd57ffff99 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py @@ -0,0 +1,39 @@ +import os + +from avalon import io +import pyblish.api +from openpype.lib import OpenPypeMongoConnection + + +class IntegrateContextToLog(pyblish.api.ContextPlugin): + """ Adds context information to log document for displaying in front end""" + + label = "Integrate Context to Log" + order = pyblish.api.IntegratorOrder - 0.1 + hosts = ["webpublisher"] + + def process(self, context): + self.log.info("Integrate Context to Log") + + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + dbcon = mongo_client[database_name]["webpublishes"] + + for instance in context: + self.log.info("ctx_path: {}".format(instance.data.get("ctx_path"))) + self.log.info("batch_id: {}".format(instance.data.get("batch_id"))) + if instance.data.get("ctx_path") and instance.data.get("batch_id"): + self.log.info("Updating log record") + dbcon.update_one( + { + "batch_id": instance.data.get("batch_id"), + "status": "in_progress" + }, + {"$set": + { + "path": instance.data.get("ctx_path") + + }} + ) + + return \ No newline at end of file From 10fa591e683eb785dea76c9ea300fe6567bdd033 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 23 Aug 2021 10:45:33 +0200 Subject: [PATCH 36/57] Webpublisher - added documentation --- openpype/cli.py | 7 ++ .../docs/admin_webserver_for_webpublisher.md | 82 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 website/docs/admin_webserver_for_webpublisher.md diff --git a/openpype/cli.py b/openpype/cli.py index 8dc32b307a6..28195008cc7 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -99,6 +99,13 @@ def eventserver(debug, @click.option("-e", "--executable", help="Executable") @click.option("-u", "--upload_dir", help="Upload dir") def webpublisherwebserver(debug, executable, upload_dir): + """Starts webserver for communication with Webpublish FR via command line + + OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND + FTRACK_BOT_API_KEY provided with api key from Ftrack. + + Expect "pype.club" user created on Ftrack. + """ if debug: os.environ['OPENPYPE_DEBUG'] = "3" diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md new file mode 100644 index 00000000000..748b269ad79 --- /dev/null +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -0,0 +1,82 @@ +--- +id: admin_webserver_for_webpublisher +title: Webserver for webpublisher +sidebar_label: Webserver for webpublisher +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Running Openpype webserver is needed as a backend part for Web publishing. +Any OS supported by Openpype could be used as a host server. + +Webpublishing consists of two sides, Front end (FE) and Openpype backend. This documenation is only targeted on OP side. + +It is expected that FE and OP will live on two separate servers, FE publicly available, OP safely in customer network. + +# Requirements for servers +- OP server allows access to its `8079` port for FE. (It is recommended to whitelist only FE IP.) +- have shared folder for published resources (images, workfiles etc) on both servers + +# Prepare Ftrack +Current webpublish process expects authentication via Slack. It is expected that customer has users created on a Ftrack +with same email addresses as on Slack. As some customer might have usernames different from emails, conversion from email to username is needed. + +For this "pype.club" user needs to be present on Ftrack, creation of this user should be standard part of Ftrack preparation for Openpype. +Next create API key on Ftrack, store this information temporarily as you won't have access to this key after creation. + + +# Prepare Openpype + +Deploy OP build distribution (Openpype Igniter) on an OS of your choice. + +##Run webserver as a Linux service: + +(This expects that OP Igniter is deployed to `opt/openpype` and log should be stored in `/tmp/openpype.log`) + +- create file `sudo vi /opt/openpype/webpublisher_webserver.sh` + +- paste content +```sh +#!/usr/bin/env bash +export OPENPYPE_DEBUG=3 +export FTRACK_BOT_API_KEY=YOUR_API_KEY +export PYTHONDONTWRITEBYTECODE=1 +export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION + +pushd /opt/openpype +./openpype_console webpublisherwebserver --upload_dir YOUR_SHARED_FOLDER_ON_HOST --executable /opt/openpype/openpype_console > /tmp/openpype.log 2>&1 +``` + +1. create service file `sudo vi /etc/systemd/system/openpye-webserver.service` + +2. paste content +```sh +[Unit] +Description=Run OpenPype Ftrack Webserver Service +After=network.target + +[Service] +Type=idle +ExecStart=/opt/openpype/webpublisher_webserver.sh +Restart=on-failure +RestartSec=10s +StandardOutput=append:/tmp/openpype.log +StandardError=append:/tmp/openpype.log + +[Install] +WantedBy=multi-user.target +``` + +5. change file permission: + `sudo chmod 0755 /etc/systemd/system/openpype-webserver.service` + +6. enable service: + `sudo systemctl enable openpype-webserver` + +7. start service: + `sudo systemctl start openpype-webserver` + +8. Check `/tmp/openpype.log` if OP got started + +(Note: service could be restarted by `service openpype-webserver restart` - this will result in purge of current log file!) \ No newline at end of file From eabfc473acc0297ec8c2b7af355acea607b98f10 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 23 Aug 2021 10:48:54 +0200 Subject: [PATCH 37/57] Hound --- .../webpublisher/plugins/publish/collect_published_files.py | 5 +++-- .../webpublisher/plugins/publish/integrate_context_to_log.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 8861190003e..59c315861e7 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -96,10 +96,11 @@ def _process_batch(self, dir_url): instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" - instance.data["ctx_path"] = ctx["path"] # to store for logging + # to store logging info into DB openpype.webpublishes + instance.data["ctx_path"] = ctx["path"] instance.data["batch_id"] = task_data["batch"] - + # to convert from email provided into Ftrack username instance.data["user_email"] = task_data["user"] if is_sequence: diff --git a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py index 1dd57ffff99..419c065e166 100644 --- a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py +++ b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py @@ -1,6 +1,5 @@ import os -from avalon import io import pyblish.api from openpype.lib import OpenPypeMongoConnection @@ -36,4 +35,4 @@ def process(self, context): }} ) - return \ No newline at end of file + return From a3f106736456755f7e7a9d4399eba68cc54d321a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 24 Aug 2021 11:08:44 +0200 Subject: [PATCH 38/57] Webpublisher - webserver ip is configurable In some cases webserver needs to listen on specific ip (because of Docker) --- openpype/modules/webserver/server.py | 5 ++++- openpype/modules/webserver/webserver_cli.py | 5 +++-- openpype/modules/webserver/webserver_module.py | 5 ++++- website/docs/admin_webserver_for_webpublisher.md | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/openpype/modules/webserver/server.py b/openpype/modules/webserver/server.py index 65c57959951..9d99e1c7a30 100644 --- a/openpype/modules/webserver/server.py +++ b/openpype/modules/webserver/server.py @@ -1,5 +1,6 @@ import threading import asyncio +import os from aiohttp import web @@ -110,7 +111,9 @@ async def start_server(self): """ Starts runner and TCPsite """ self.runner = web.AppRunner(self.manager.app) await self.runner.setup() - self.site = web.TCPSite(self.runner, 'localhost', self.port) + host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' + log.info("host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) + self.site = web.TCPSite(self.runner, host_ip, self.port) await self.site.start() def stop(self): diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 8e4dfd229d2..24bd28ba7d3 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -19,7 +19,7 @@ from openpype.api import get_system_settings -SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost +# SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost log = PypeLogger().get_logger("webserver_gui") @@ -129,7 +129,8 @@ def reprocess_failed(upload_dir): }} ) continue - server_url = "{}/api/webpublish/batch".format(SERVER_URL) + server_url = "{}/api/webpublish/batch".format( + os.environ["OPENPYPE_WEBSERVER_URL"]) with open(batch_url) as f: data = json.loads(f.read()) diff --git a/openpype/modules/webserver/webserver_module.py b/openpype/modules/webserver/webserver_module.py index 4832038575a..10508265da3 100644 --- a/openpype/modules/webserver/webserver_module.py +++ b/openpype/modules/webserver/webserver_module.py @@ -79,7 +79,10 @@ def create_server_manager(self): self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) - webserver_url = "http://localhost:{}".format(self.port) + # in a case that webserver should listen on specific ip (webpublisher) + self.log.info("module host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) + host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' + webserver_url = "http://{}:{}".format(host_ip, self.port) os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url @staticmethod diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md index 748b269ad79..2b23033595b 100644 --- a/website/docs/admin_webserver_for_webpublisher.md +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -40,6 +40,7 @@ Deploy OP build distribution (Openpype Igniter) on an OS of your choice. ```sh #!/usr/bin/env bash export OPENPYPE_DEBUG=3 +export WEBSERVER_HOST_IP=localhost export FTRACK_BOT_API_KEY=YOUR_API_KEY export PYTHONDONTWRITEBYTECODE=1 export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION From 47f529bdccf00aebccf83971403b4ca91dc9bdb8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 10:50:35 +0200 Subject: [PATCH 39/57] Webpublisher - rename to last version --- .../webpublisher/plugins/publish/collect_published_files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 59c315861e7..6584120d973 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -92,7 +92,8 @@ def _process_batch(self, dir_url): instance.data["subset"] = subset instance.data["family"] = family instance.data["families"] = families - instance.data["version"] = self._get_version(asset, subset) + 1 + instance.data["version"] = \ + self._get_last_version(asset, subset) + 1 instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" @@ -195,7 +196,7 @@ def _get_family(self, settings, task_type, is_sequence, extension): content["subset_template_name"], \ content["tags"] - def _get_version(self, asset_name, subset_name): + def _get_last_version(self, asset_name, subset_name): """Returns version number or 0 for 'asset' and 'subset'""" query = [ { From 5164481b367bcfb1a7fc768d5b83a3f7b896ac0b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 10:57:22 +0200 Subject: [PATCH 40/57] Webpublisher - introduced FTRACK_BOT_API_USER --- openpype/hosts/webpublisher/plugins/publish/collect_username.py | 2 +- website/docs/admin_webserver_for_webpublisher.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/hosts/webpublisher/plugins/publish/collect_username.py index 0c2c6310f4e..7a303a16088 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_username.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_username.py @@ -31,7 +31,7 @@ class CollectUsername(pyblish.api.ContextPlugin): _context = None def process(self, context): - os.environ["FTRACK_API_USER"] = "pype.club" + os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"] os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] self.log.info("CollectUsername") for instance in context: diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md index 2b23033595b..dced825bdcc 100644 --- a/website/docs/admin_webserver_for_webpublisher.md +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -41,6 +41,7 @@ Deploy OP build distribution (Openpype Igniter) on an OS of your choice. #!/usr/bin/env bash export OPENPYPE_DEBUG=3 export WEBSERVER_HOST_IP=localhost +export FTRACK_BOT_API_USER=YOUR_API_USER export FTRACK_BOT_API_KEY=YOUR_API_KEY export PYTHONDONTWRITEBYTECODE=1 export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION From bcea6fbf0d2b19ae92fc946ff1501704ce024308 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:09:57 +0200 Subject: [PATCH 41/57] Webpublisher - removed is_webpublish_enabled as unneeded run_server gets triggered only for webpublisher, doesn't make sense to double check Moved webpublish dependent classes under webpublish host Cleaned up setting --- .../webserver_service}/webpublish_routes.py | 0 .../webserver_service}/webserver_cli.py | 115 ++++++++---------- openpype/pype_commands.py | 7 +- .../defaults/system_settings/modules.json | 3 - .../schemas/system_schema/schema_modules.json | 14 --- 5 files changed, 55 insertions(+), 84 deletions(-) rename openpype/{modules/webserver => hosts/webpublisher/webserver_service}/webpublish_routes.py (100%) rename openpype/{modules/webserver => hosts/webpublisher/webserver_service}/webserver_cli.py (52%) diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py similarity index 100% rename from openpype/modules/webserver/webpublish_routes.py rename to openpype/hosts/webpublisher/webserver_service/webpublish_routes.py diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py similarity index 52% rename from openpype/modules/webserver/webserver_cli.py rename to openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 24bd28ba7d3..b1c14260e93 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -19,7 +19,6 @@ from openpype.api import get_system_settings -# SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost log = PypeLogger().get_logger("webserver_gui") @@ -32,72 +31,62 @@ def run_webserver(*args, **kwargs): webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - is_webpublish_enabled = False - webpublish_tool = get_system_settings()["modules"].\ - get("webpublish_tool") - - if webpublish_tool and webpublish_tool["enabled"]: - is_webpublish_enabled = True - - log.debug("is_webpublish_enabled {}".format(is_webpublish_enabled)) - if is_webpublish_enabled: - resource = RestApiResource(webserver_module.server_manager, - upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"]) - projects_endpoint = WebpublisherProjectsEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/projects", - projects_endpoint.dispatch - ) - - hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/hierarchy/{project_name}", - hiearchy_endpoint.dispatch - ) - - # triggers publish - webpublisher_task_publish_endpoint = \ - WebpublisherBatchPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/batch", - webpublisher_task_publish_endpoint.dispatch - ) - - webpublisher_batch_publish_endpoint = \ - WebpublisherTaskPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/task", - webpublisher_batch_publish_endpoint.dispatch - ) - - # reporting - openpype_resource = OpenPypeRestApiResource() - batch_status_endpoint = BatchStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/batch_status/{batch_id}", - batch_status_endpoint.dispatch - ) - - user_status_endpoint = PublishesStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/publishes/{user}", - user_status_endpoint.dispatch - ) + resource = RestApiResource(webserver_module.server_manager, + upload_dir=kwargs["upload_dir"], + executable=kwargs["executable"]) + projects_endpoint = WebpublisherProjectsEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/projects", + projects_endpoint.dispatch + ) + + hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/hierarchy/{project_name}", + hiearchy_endpoint.dispatch + ) + + # triggers publish + webpublisher_task_publish_endpoint = \ + WebpublisherBatchPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/batch", + webpublisher_task_publish_endpoint.dispatch + ) + + webpublisher_batch_publish_endpoint = \ + WebpublisherTaskPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/task", + webpublisher_batch_publish_endpoint.dispatch + ) + + # reporting + openpype_resource = OpenPypeRestApiResource() + batch_status_endpoint = BatchStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/batch_status/{batch_id}", + batch_status_endpoint.dispatch + ) + + user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/publishes/{user}", + user_status_endpoint.dispatch + ) webserver_module.start_server() last_reprocessed = time.time() while True: - if is_webpublish_enabled: - if time.time() - last_reprocessed > 20: - reprocess_failed(kwargs["upload_dir"]) - last_reprocessed = time.time() + if time.time() - last_reprocessed > 20: + reprocess_failed(kwargs["upload_dir"]) + last_reprocessed = time.time() time.sleep(1.0) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d288e9f2a3e..e0cab962f69 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -42,9 +42,8 @@ def launch_eventservercli(*args): @staticmethod def launch_webpublisher_webservercli(*args, **kwargs): - from openpype.modules.webserver.webserver_cli import ( - run_webserver - ) + from openpype.hosts.webpublisher.webserver_service.webserver_cli \ + import (run_webserver) return run_webserver(*args, **kwargs) @staticmethod @@ -53,7 +52,7 @@ def launch_standalone_publisher(): standalonepublish.main() @staticmethod - def publish(paths, targets=None, host=None): + def publish(paths, targets=None): """Start headless publishing. Publish use json from passed paths argument. diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 1005f8d16bf..3a70b905901 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -167,9 +167,6 @@ "standalonepublish_tool": { "enabled": true }, - "webpublish_tool": { - "enabled": false - }, "project_manager": { "enabled": true }, diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 8cd729d2a1b..75c08b2cd97 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -197,20 +197,6 @@ } ] }, - { - "type": "dict", - "key": "webpublish_tool", - "label": "Web Publish", - "collapsible": true, - "checkbox_key": "enabled", - "children": [ - { - "type": "boolean", - "key": "enabled", - "label": "Enabled" - } - ] - }, { "type": "dict", "key": "project_manager", From 9a9acc119ae5a95b117cca0ad6ffad65554faa71 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:23:26 +0200 Subject: [PATCH 42/57] Webpublisher - introduced command line arguments for host and port --- openpype/cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 28195008cc7..0b6d41b060a 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -96,9 +96,11 @@ def eventserver(debug, @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("-h", "--host", help="Host", default=None) +@click.option("-p", "--port", help="Port", default=None) @click.option("-e", "--executable", help="Executable") @click.option("-u", "--upload_dir", help="Upload dir") -def webpublisherwebserver(debug, executable, upload_dir): +def webpublisherwebserver(debug, executable, upload_dir, host=None, port=None): """Starts webserver for communication with Webpublish FR via command line OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND @@ -111,7 +113,9 @@ def webpublisherwebserver(debug, executable, upload_dir): PypeCommands().launch_webpublisher_webservercli( upload_dir=upload_dir, - executable=executable + executable=executable, + host=host, + port=port ) From 91f9362288b1a6a4ca5d894812e6e32621f5874c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:47:52 +0200 Subject: [PATCH 43/57] Webpublisher - proper merge --- .../default_modules/webserver/server.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/modules/default_modules/webserver/server.py b/openpype/modules/default_modules/webserver/server.py index 9d99e1c7a30..83a29e074e8 100644 --- a/openpype/modules/default_modules/webserver/server.py +++ b/openpype/modules/default_modules/webserver/server.py @@ -1,6 +1,5 @@ import threading import asyncio -import os from aiohttp import web @@ -11,8 +10,9 @@ class WebServerManager: """Manger that care about web server thread.""" - def __init__(self, module): - self.module = module + def __init__(self, port=None, host=None): + self.port = port or 8079 + self.host = host or "localhost" self.client = None self.handlers = {} @@ -25,8 +25,8 @@ def __init__(self, module): self.webserver_thread = WebServerThread(self) @property - def port(self): - return self.module.port + def url(self): + return "http://{}:{}".format(self.host, self.port) def add_route(self, *args, **kwargs): self.app.router.add_route(*args, **kwargs) @@ -79,6 +79,10 @@ def __init__(self, manager): def port(self): return self.manager.port + @property + def host(self): + return self.manager.host + def run(self): self.is_running = True @@ -111,9 +115,7 @@ async def start_server(self): """ Starts runner and TCPsite """ self.runner = web.AppRunner(self.manager.app) await self.runner.setup() - host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' - log.info("host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) - self.site = web.TCPSite(self.runner, host_ip, self.port) + self.site = web.TCPSite(self.runner, self.host, self.port) await self.site.start() def stop(self): From f7cb778470fbfbab3b0ad7209912670f0992cca2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:48:43 +0200 Subject: [PATCH 44/57] Webpublisher - proper merge --- .../webserver/webserver_module.py | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index c000d5ce10c..bdb0010118c 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -1,24 +1,34 @@ import os import socket +from abc import ABCMeta, abstractmethod + +import six from openpype import resources -from openpype.modules import OpenPypeModule -from openpype_interfaces import ( - ITrayService, - IWebServerRoutes -) +from .. import PypeModule, ITrayService + +@six.add_metaclass(ABCMeta) +class IWebServerRoutes: + """Other modules interface to register their routes.""" + @abstractmethod + def webserver_initialization(self, server_manager): + pass -class WebServerModule(OpenPypeModule, ITrayService): + +class WebServerModule(PypeModule, ITrayService): name = "webserver" label = "WebServer" + webserver_url_env = "OPENPYPE_WEBSERVER_URL" + def initialize(self, _module_settings): self.enabled = True self.server_manager = None self._host_listener = None self.port = self.find_free_port() + self.webserver_url = None def connect_with_modules(self, enabled_modules): if not self.server_manager: @@ -44,7 +54,7 @@ def _add_resources_statics(self): self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( - os.environ["OPENPYPE_WEBSERVER_URL"], static_prefix + self.webserver_url, static_prefix ) def _add_listeners(self): @@ -62,21 +72,33 @@ def stop_server(self): if self.server_manager: self.server_manager.stop_server() + @staticmethod + def create_new_server_manager(port=None, host=None): + """Create webserver manager for passed port and host. + + Args: + port(int): Port on which wil webserver listen. + host(str): Host name or IP address. Default is 'localhost'. + + Returns: + WebServerManager: Prepared manager. + """ + from .server import WebServerManager + + return WebServerManager(port, host) + def create_server_manager(self): if self.server_manager: return - from .server import WebServerManager - - self.server_manager = WebServerManager(self) + self.server_manager = self.create_new_server_manager(self.port) self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) - # in a case that webserver should listen on specific ip (webpublisher) - self.log.info("module host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) - host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' - webserver_url = "http://{}:{}".format(host_ip, self.port) - os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url + + webserver_url = self.server_manager.url + os.environ[self.webserver_url_env] = str(webserver_url) + self.webserver_url = webserver_url @staticmethod def find_free_port( From b235068a3eee5858be38cd8c70ed9bb4d824d2ae Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:55:32 +0200 Subject: [PATCH 45/57] Webpublisher - proper merge --- .../webserver_service/webpublish_routes.py | 2 +- .../default_modules/webserver/webserver_module.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 32feb276ed6..0014d1b3449 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -10,7 +10,7 @@ from avalon.api import AvalonMongoDB from openpype.lib import OpenPypeMongoConnection -from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint +from openpype_modules.avalon_apps.rest_api import _RestApiEndpoint from openpype.lib import PypeLogger diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index bdb0010118c..d8e54632b5b 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -5,7 +5,11 @@ import six from openpype import resources -from .. import PypeModule, ITrayService +from openpype.modules import OpenPypeModule +from openpype_interfaces import ( + ITrayService, + IWebServerRoutes +) @six.add_metaclass(ABCMeta) @@ -16,7 +20,7 @@ def webserver_initialization(self, server_manager): pass -class WebServerModule(PypeModule, ITrayService): +class WebServerModule(OpenPypeModule, ITrayService): name = "webserver" label = "WebServer" @@ -53,6 +57,8 @@ def _add_resources_statics(self): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) + webserver_url = "http://localhost:{}".format(self.port) + os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( self.webserver_url, static_prefix ) From e666fad275a018aad670418605e041614eb85e1c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:57:58 +0200 Subject: [PATCH 46/57] Webpublisher - updated help label --- openpype/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/cli.py b/openpype/cli.py index 0b6d41b060a..c446d5e4432 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -162,7 +162,7 @@ def publish(debug, paths, targets): @click.option("-h", "--host", help="Host") @click.option("-u", "--user", help="User email address") @click.option("-p", "--project", help="Project") -@click.option("-t", "--targets", help="Targets module", default=None, +@click.option("-t", "--targets", help="Targets", default=None, multiple=True) def remotepublish(debug, project, path, host, targets=None, user=None): """Start CLI publishing. From 1387b75d5f4d1242c2ab7f35f8e406564013b848 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 12:36:03 +0200 Subject: [PATCH 47/57] Webpublisher - revert mixed up commit --- openpype/lib/applications.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 19208ff1736..71ab2eac61b 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -1009,7 +1009,7 @@ def __init__(self, data): def get_app_environments_for_context( - project_name, asset_name, task_name, app_name=None, env=None + project_name, asset_name, task_name, app_name, env=None ): """Prepare environment variables by context. Args: @@ -1038,14 +1038,20 @@ def get_app_environments_for_context( "name": asset_name }) + # Prepare app object which can be obtained only from ApplciationManager + app_manager = ApplicationManager() + app = app_manager.applications[app_name] + # Project's anatomy anatomy = Anatomy(project_name) - prep_dict = { + data = EnvironmentPrepData({ "project_name": project_name, "asset_name": asset_name, "task_name": task_name, + "app": app, + "dbcon": dbcon, "project_doc": project_doc, "asset_doc": asset_doc, @@ -1053,15 +1059,7 @@ def get_app_environments_for_context( "anatomy": anatomy, "env": env - } - - if app_name: - # Prepare app object which can be obtained only from ApplicationManager - app_manager = ApplicationManager() - app = app_manager.applications[app_name] - prep_dict["app"] = app - - data = EnvironmentPrepData(prep_dict) + }) prepare_host_environments(data) prepare_context_environments(data) From 3dfe3513c3e44e445042c086ea94740c542a8e3b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 13:10:45 +0200 Subject: [PATCH 48/57] Webpublisher - fixed host install --- openpype/hosts/webpublisher/__init__.py | 3 +++ openpype/hosts/webpublisher/api/__init__.py | 1 - openpype/pype_commands.py | 9 +++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index e69de29bb2d..d47bab580bd 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -0,0 +1,3 @@ +# to have required methods for interface +def ls(): + pass \ No newline at end of file diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 1bf1ef1a6f1..76709bb2d74 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -29,7 +29,6 @@ def install(): log.info(PUBLISH_PATH) io.install() - avalon.Session["AVALON_APP"] = "webpublisher" # because of Ftrack collect avalon.on("application.launched", application_launch) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 7774a010a6e..656f8642297 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -157,11 +157,12 @@ def remotepublish(project, batch_path, host, user, targets=None): os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host # to trigger proper plugings + os.environ["AVALON_APP"] = host - # this should be more generic - from openpype.hosts.webpublisher.api import install as w_install - w_install() + import avalon.api + from openpype.hosts import webpublisher + + avalon.api.install(webpublisher) log.info("Running publish ...") From c7c45ecf879ffe54ee7f942a7490350baec0c862 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 13:24:42 +0200 Subject: [PATCH 49/57] Webpublisher - removed unwanted folder --- openpype/modules/sync_server/__init__.py | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 openpype/modules/sync_server/__init__.py diff --git a/openpype/modules/sync_server/__init__.py b/openpype/modules/sync_server/__init__.py deleted file mode 100644 index a814f0db622..00000000000 --- a/openpype/modules/sync_server/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from openpype.modules.sync_server.sync_server_module import SyncServerModule - - -def tray_init(tray_widget, main_widget): - return SyncServerModule() From a65f0e15d71632ff94dc90d3a8b13222149f494d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 14:00:09 +0200 Subject: [PATCH 50/57] fixed webserver module --- .../default_modules/webserver/webserver_module.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index d8e54632b5b..cfbb0c1ee08 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -12,14 +12,6 @@ ) -@six.add_metaclass(ABCMeta) -class IWebServerRoutes: - """Other modules interface to register their routes.""" - @abstractmethod - def webserver_initialization(self, server_manager): - pass - - class WebServerModule(OpenPypeModule, ITrayService): name = "webserver" label = "WebServer" @@ -57,8 +49,6 @@ def _add_resources_statics(self): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) - webserver_url = "http://localhost:{}".format(self.port) - os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( self.webserver_url, static_prefix ) From 5dbc7ab36d1eb3c601bbb6cffd92a6a0624d4852 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 14:02:55 +0200 Subject: [PATCH 51/57] recommit changes --- .../webserver_service/webserver_cli.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index b1c14260e93..b733cc260fa 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -29,20 +29,23 @@ def run_webserver(*args, **kwargs): manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - webserver_module.create_server_manager() + host = os.environ.get("WEBSERVER_HOST_IP") + port = 8079 + server_manager = webserver_module.create_new_server_manager(port, host) + webserver_url = server_manager.url - resource = RestApiResource(webserver_module.server_manager, + resource = RestApiResource(server_manager, upload_dir=kwargs["upload_dir"], executable=kwargs["executable"]) projects_endpoint = WebpublisherProjectsEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/projects", projects_endpoint.dispatch ) hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/hierarchy/{project_name}", hiearchy_endpoint.dispatch @@ -51,7 +54,7 @@ def run_webserver(*args, **kwargs): # triggers publish webpublisher_task_publish_endpoint = \ WebpublisherBatchPublishEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "POST", "/api/webpublish/batch", webpublisher_task_publish_endpoint.dispatch @@ -59,7 +62,7 @@ def run_webserver(*args, **kwargs): webpublisher_batch_publish_endpoint = \ WebpublisherTaskPublishEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "POST", "/api/webpublish/task", webpublisher_batch_publish_endpoint.dispatch @@ -68,29 +71,29 @@ def run_webserver(*args, **kwargs): # reporting openpype_resource = OpenPypeRestApiResource() batch_status_endpoint = BatchStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/batch_status/{batch_id}", batch_status_endpoint.dispatch ) user_status_endpoint = PublishesStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/publishes/{user}", user_status_endpoint.dispatch ) - webserver_module.start_server() + server_manager.start_server() last_reprocessed = time.time() while True: if time.time() - last_reprocessed > 20: - reprocess_failed(kwargs["upload_dir"]) + reprocess_failed(kwargs["upload_dir"], webserver_url) last_reprocessed = time.time() time.sleep(1.0) -def reprocess_failed(upload_dir): +def reprocess_failed(upload_dir, webserver_url): # log.info("check_reprocesable_records") from openpype.lib import OpenPypeMongoConnection @@ -118,8 +121,7 @@ def reprocess_failed(upload_dir): }} ) continue - server_url = "{}/api/webpublish/batch".format( - os.environ["OPENPYPE_WEBSERVER_URL"]) + server_url = "{}/api/webpublish/batch".format(webserver_url) with open(batch_url) as f: data = json.loads(f.read()) From cd5659248bc6db9c4a4602b24ddb467415f188c2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 14:10:34 +0200 Subject: [PATCH 52/57] use host port from kwargs --- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index b733cc260fa..06d78e2fca7 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -29,8 +29,8 @@ def run_webserver(*args, **kwargs): manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - host = os.environ.get("WEBSERVER_HOST_IP") - port = 8079 + host = kwargs.get("host") or "localhost" + port = kwargs.get("port") or 8079 server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url From 66b710116447774fee71f088d34c8eb5e34ab798 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 14:11:08 +0200 Subject: [PATCH 53/57] Webpublisher - fix propagation of host --- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index b733cc260fa..723762003d9 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -29,8 +29,8 @@ def run_webserver(*args, **kwargs): manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - host = os.environ.get("WEBSERVER_HOST_IP") - port = 8079 + host = kwargs["host"] + port = kwargs["port"] server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url From 2d55233c6b2ab4c8ed8129f052b08eda70a3bffe Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 14:33:16 +0200 Subject: [PATCH 54/57] Hound --- openpype/hosts/webpublisher/__init__.py | 2 +- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 +--- .../modules/default_modules/webserver/webserver_module.py | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index d47bab580bd..3de2e3434b6 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -1,3 +1,3 @@ # to have required methods for interface def ls(): - pass \ No newline at end of file + pass diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 06d78e2fca7..d00d269059f 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -17,8 +17,6 @@ PublishesStatusEndpoint ) -from openpype.api import get_system_settings - log = PypeLogger().get_logger("webserver_gui") @@ -129,7 +127,7 @@ def reprocess_failed(upload_dir, webserver_url): try: r = requests.post(server_url, json=data) log.info("response{}".format(r)) - except: + except Exception: log.info("exception", exc_info=True) dbcon.update_one( diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index cfbb0c1ee08..5bfb2d63907 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -1,8 +1,5 @@ import os import socket -from abc import ABCMeta, abstractmethod - -import six from openpype import resources from openpype.modules import OpenPypeModule From 430801da30a7fa259bcd1b815e643c5433a41651 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 14:36:37 +0200 Subject: [PATCH 55/57] Webpublisher - move plugin to Ftrack --- .../default_modules/ftrack}/plugins/publish/collect_username.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openpype/{hosts/webpublisher => modules/default_modules/ftrack}/plugins/publish/collect_username.py (100%) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py similarity index 100% rename from openpype/hosts/webpublisher/plugins/publish/collect_username.py rename to openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py From 8c9f20bbc483d0facc9e8a40e6789dbf7fa2e9e9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 15:06:27 +0200 Subject: [PATCH 56/57] Webpublisher - moved dummy ls to api --- openpype/hosts/webpublisher/__init__.py | 3 --- openpype/hosts/webpublisher/api/__init__.py | 5 +++++ openpype/pype_commands.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index 3de2e3434b6..e69de29bb2d 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -1,3 +0,0 @@ -# to have required methods for interface -def ls(): - pass diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 76709bb2d74..e40d46d6627 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -36,3 +36,8 @@ def uninstall(): pyblish.deregister_plugin_path(PUBLISH_PATH) avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH) avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH) + + +# to have required methods for interface +def ls(): + pass diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 656f8642297..c18fe366672 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -160,7 +160,7 @@ def remotepublish(project, batch_path, host, user, targets=None): os.environ["AVALON_APP"] = host import avalon.api - from openpype.hosts import webpublisher + from openpype.hosts.webpublisher import api as webpublisher avalon.api.install(webpublisher) From cb229e9185308fc08603c88341c8792fd63f652b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 15:11:08 +0200 Subject: [PATCH 57/57] Merge back to develop --- repos/avalon-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/avalon-core b/repos/avalon-core index 82d5b8137ee..52e24a9993e 160000 --- a/repos/avalon-core +++ b/repos/avalon-core @@ -1 +1 @@ -Subproject commit 82d5b8137eea3b49d4781a4af51d7f375bb9f628 +Subproject commit 52e24a9993e5223b0a719786e77a4b87e936e556