From 573cb3d1b30426b998be8b55d7176f906b88f776 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 18 Jan 2022 15:16:39 +0100 Subject: [PATCH 01/11] OP-2427 - added BatchReprocessEndpoint Endpoint marks latest batch with status 'error' for reprocessing. --- .../webserver_service/webpublish_routes.py | 31 +++++++++++++++++++ .../webserver_service/webserver_cli.py | 9 ++++++ 2 files changed, 40 insertions(+) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 30399a6ba76..ff4c5830985 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -351,3 +351,34 @@ async def get(self, project_name=None) -> Response: body=self.resource.encode(dict(configured)), content_type="application/json" ) + + +class BatchReprocessEndpoint(_RestApiEndpoint): + """Marks latest 'batch_id' for reprocessing, returns 404 if not found.""" + async def post(self, batch_id) -> Response: + batches = self.dbcon.find({"batch_id": batch_id, + "status": "error"}).sort("_id", -1) + batch = None + if batches: + batch = batches[0] + + if batch: + self.dbcon.update_one( + {"_id": batch["_id"]}, + {"$set": + { + "status": "reprocess" + }} + ) + output = [{"msg": "Batch id {} set to reprocess".format(batch_id)}] + status = 200 + else: + output = [{"msg": "Batch id {} not found".format(batch_id)}] + status = 404 + body = self.resource.encode(output) + + return Response( + status=status, + body=body, + content_type="application/json" + ) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index c96ad8e110a..374e7e80a5b 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -12,6 +12,7 @@ RestApiResource, OpenPypeRestApiResource, WebpublisherBatchPublishEndpoint, + BatchReprocessEndpoint, WebpublisherTaskPublishEndpoint, WebpublisherHiearchyEndpoint, WebpublisherProjectsEndpoint, @@ -95,6 +96,14 @@ def run_webserver(*args, **kwargs): user_status_endpoint.dispatch ) + webpublisher_batch_reprocess_endpoint = \ + BatchReprocessEndpoint(openpype_resource) + server_manager.add_route( + "POST", + "/api/webpublish/reprocess/{batch_id}", + webpublisher_batch_reprocess_endpoint.dispatch + ) + server_manager.start_server() last_reprocessed = time.time() while True: From 26c5eac8b094a322c26097869fa116e491021c74 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 18 Jan 2022 15:25:45 +0100 Subject: [PATCH 02/11] OP-2427 - refactored names --- .../webserver_service/webpublish_routes.py | 10 +++++----- .../webserver_service/webserver_cli.py | 20 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index ff4c5830985..6606f5cb589 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -61,7 +61,7 @@ def __init__(self, ): self.dbcon = mongo_client[database_name]["webpublishes"] -class WebpublisherProjectsEndpoint(_RestApiEndpoint): +class ProjectsEndpoint(_RestApiEndpoint): """Returns list of dict with project info (id, name).""" async def get(self) -> Response: output = [] @@ -82,7 +82,7 @@ async def get(self) -> Response: ) -class WebpublisherHiearchyEndpoint(_RestApiEndpoint): +class HiearchyEndpoint(_RestApiEndpoint): """Returns dictionary with context tree from assets.""" async def get(self, project_name) -> Response: query_projection = { @@ -181,7 +181,7 @@ def __init__(self, node_type, name): self["attributes"] = {} -class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): +class BatchPublishEndpoint(_RestApiEndpoint): """Triggers headless publishing of batch.""" async def post(self, request) -> Response: # Validate existence of openpype executable @@ -190,7 +190,7 @@ async def post(self, request) -> Response: msg = "Non existent OpenPype executable {}".format(openpype_app) raise RuntimeError(msg) - log.info("WebpublisherBatchPublishEndpoint called") + log.info("BatchPublishEndpoint called") content = await request.json() # Each filter have extensions which are checked on first task item @@ -286,7 +286,7 @@ async def post(self, request) -> Response: ) -class WebpublisherTaskPublishEndpoint(_RestApiEndpoint): +class TaskPublishEndpoint(_RestApiEndpoint): """Prepared endpoint triggered after each task - for future development.""" async def post(self, request) -> Response: return Response( diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 374e7e80a5b..a88b4bbc3e8 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -11,14 +11,14 @@ from .webpublish_routes import ( RestApiResource, OpenPypeRestApiResource, - WebpublisherBatchPublishEndpoint, + HiearchyEndpoint, + ProjectsEndpoint, + ConfiguredExtensionsEndpoint, + BatchPublishEndpoint, BatchReprocessEndpoint, - WebpublisherTaskPublishEndpoint, - WebpublisherHiearchyEndpoint, - WebpublisherProjectsEndpoint, BatchStatusEndpoint, - PublishesStatusEndpoint, - ConfiguredExtensionsEndpoint + TaskPublishEndpoint, + PublishesStatusEndpoint ) @@ -42,14 +42,14 @@ def run_webserver(*args, **kwargs): upload_dir=kwargs["upload_dir"], executable=kwargs["executable"], studio_task_queue=studio_task_queue) - projects_endpoint = WebpublisherProjectsEndpoint(resource) + projects_endpoint = ProjectsEndpoint(resource) server_manager.add_route( "GET", "/api/projects", projects_endpoint.dispatch ) - hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + hiearchy_endpoint = HiearchyEndpoint(resource) server_manager.add_route( "GET", "/api/hierarchy/{project_name}", @@ -65,7 +65,7 @@ def run_webserver(*args, **kwargs): # triggers publish webpublisher_task_publish_endpoint = \ - WebpublisherBatchPublishEndpoint(resource) + BatchPublishEndpoint(resource) server_manager.add_route( "POST", "/api/webpublish/batch", @@ -73,7 +73,7 @@ def run_webserver(*args, **kwargs): ) webpublisher_batch_publish_endpoint = \ - WebpublisherTaskPublishEndpoint(resource) + TaskPublishEndpoint(resource) server_manager.add_route( "POST", "/api/webpublish/task", From 5a3a6cc6e8154cfe9fe473b79aaa375806c801ac Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 13:23:54 +0100 Subject: [PATCH 03/11] OP-2427 - better error handling when batch or user not found --- .../webserver_service/webpublish_routes.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 6606f5cb589..e5ae727cd7c 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -301,9 +301,16 @@ class BatchStatusEndpoint(_RestApiEndpoint): async def get(self, batch_id) -> Response: output = self.dbcon.find_one({"batch_id": batch_id}) + if output: + status = 200 + body = self.resource.encode(output) + else: + output = [{"msg": "Batch id {} not found".format(batch_id)}] + status = 404 + return Response( - status=200, - body=self.resource.encode(output), + status=status, + body=body, content_type="application/json" ) @@ -313,9 +320,16 @@ class PublishesStatusEndpoint(_RestApiEndpoint): async def get(self, user) -> Response: output = list(self.dbcon.find({"user": user})) + if output: + status = 200 + body = self.resource.encode(output) + else: + body = [{"msg": "User {} not found".format(user)}] + status = 404 + return Response( - status=200, - body=self.resource.encode(output), + status=status, + body=body, content_type="application/json" ) From 3328a80a21c2f5833c895a3f22b8eca44a9e9139 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 13:29:43 +0100 Subject: [PATCH 04/11] OP-2427 - better error handling when batch or user not found --- .../webpublisher/webserver_service/webpublish_routes.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index e5ae727cd7c..3270fe8f273 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -303,11 +303,10 @@ async def get(self, batch_id) -> Response: if output: status = 200 - body = self.resource.encode(output) else: - output = [{"msg": "Batch id {} not found".format(batch_id)}] + output = {"msg": "Batch id {} not found".format(batch_id)} status = 404 - + body = self.resource.encode(output) return Response( status=status, body=body, @@ -322,10 +321,10 @@ async def get(self, user) -> Response: if output: status = 200 - body = self.resource.encode(output) else: - body = [{"msg": "User {} not found".format(user)}] + output = {"msg": "User {} not found".format(user)} status = 404 + body = self.resource.encode(output) return Response( status=status, From d1d2ecb35294be89564ed27b421df07643070795 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 16:54:28 +0100 Subject: [PATCH 05/11] OP-2427 - limit sent data for user report --- .../hosts/webpublisher/webserver_service/webpublish_routes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 3270fe8f273..f55c8dec4c7 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -317,7 +317,8 @@ async def get(self, batch_id) -> Response: class PublishesStatusEndpoint(_RestApiEndpoint): """Returns list of dict with batch info for user (email address).""" async def get(self, user) -> Response: - output = list(self.dbcon.find({"user": user})) + output = list(self.dbcon.find({"user": user}, + projection={"log": False})) if output: status = 200 From 99413c26bbd44bc8e387e27a4ce69d659b1d92f2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 16:55:32 +0100 Subject: [PATCH 06/11] OP-2427 - refactor name --- .../hosts/webpublisher/webserver_service/webpublish_routes.py | 2 +- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index f55c8dec4c7..808d77a4e35 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -314,7 +314,7 @@ async def get(self, batch_id) -> Response: ) -class PublishesStatusEndpoint(_RestApiEndpoint): +class UserReportEndpoint(_RestApiEndpoint): """Returns list of dict with batch info for user (email address).""" async def get(self, user) -> Response: output = list(self.dbcon.find({"user": user}, diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index a88b4bbc3e8..12086345445 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -18,7 +18,7 @@ BatchReprocessEndpoint, BatchStatusEndpoint, TaskPublishEndpoint, - PublishesStatusEndpoint + UserReportEndpoint ) @@ -89,7 +89,7 @@ def run_webserver(*args, **kwargs): batch_status_endpoint.dispatch ) - user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + user_status_endpoint = UserReportEndpoint(openpype_resource) server_manager.add_route( "GET", "/api/publishes/{user}", From dd1c0b18dfed42926cf343a8c4586b8564f14d08 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 17:02:29 +0100 Subject: [PATCH 07/11] OP-2427 - refactor code --- .../webserver_service/webpublish_routes.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 808d77a4e35..099819f6cac 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -372,17 +372,11 @@ class BatchReprocessEndpoint(_RestApiEndpoint): async def post(self, batch_id) -> Response: batches = self.dbcon.find({"batch_id": batch_id, "status": "error"}).sort("_id", -1) - batch = None - if batches: - batch = batches[0] - if batch: + if batches: self.dbcon.update_one( - {"_id": batch["_id"]}, - {"$set": - { - "status": "reprocess" - }} + {"_id": batches[0]["_id"]}, + {"$set": {"status": "reprocess"}} ) output = [{"msg": "Batch id {} set to reprocess".format(batch_id)}] status = 200 From 769dc41ebb5b5630c22ee2fa34fcf02b7bc4d854 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 17:38:58 +0100 Subject: [PATCH 08/11] OP-2427 - reprocess all failed records for batch only once --- .../webserver_service/webserver_cli.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 45338a59257..0d0a0223d90 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -126,7 +126,11 @@ def reprocess_failed(upload_dir, webserver_url): dbcon = mongo_client[database_name]["webpublishes"] results = dbcon.find({"status": "reprocess"}) + reprocessed_batches = set() for batch in results: + if batch["batch_id"] in reprocessed_batches: + continue + batch_url = os.path.join(upload_dir, batch["batch_id"], "manifest.json") @@ -150,18 +154,24 @@ def reprocess_failed(upload_dir, webserver_url): with open(batch_url) as f: data = json.loads(f.read()) + dbcon.update_many( + { + "batch_id": batch["batch_id"], + "status": {"$in": ["error", "reprocess"]} + }, + { + "$set": { + "finish_date": datetime.now(), + "status": "sent_for_reprocessing", + "progress": 100 + } + } + ) + try: r = requests.post(server_url, json=data) log.info("response{}".format(r)) except Exception: log.info("exception", exc_info=True) - dbcon.update_one( - {"_id": batch["_id"]}, - {"$set": - { - "finish_date": datetime.now(), - "status": "sent_for_reprocessing", - "progress": 100 - }} - ) + reprocessed_batches.add(batch["batch_id"]) From 770a6407c499b751e08b3156ff4b92ed1159a6ff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 18:14:43 +0100 Subject: [PATCH 09/11] OP-2427 - mark republished records --- openpype/lib/remote_publish.py | 35 ++++++++++++++++++++++++---------- openpype/pype_commands.py | 2 +- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index dd5a3e2864e..181802792af 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -26,7 +26,7 @@ def headless_publish(log, close_plugin_name=None, is_test=False): "batch will be unfinished!") return - publish_and_log(dbcon, _id, log, close_plugin_name) + publish_and_log(dbcon, _id, log, close_plugin_name=close_plugin_name) else: publish(log, close_plugin_name) @@ -84,13 +84,14 @@ def publish(log, close_plugin_name=None): sys.exit(1) -def publish_and_log(dbcon, _id, log, close_plugin_name=None): +def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): """Loops through all plugins, logs ok and fails into OP DB. Args: dbcon (OpenPypeMongoConnection) - _id (str) + _id (str) - id of current job in DB log (OpenPypeLogger) + batch_id (str) - id sent from frontend close_plugin_name (str): name of plugin with responsibility to close host app """ @@ -143,15 +144,29 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None): ) # final update + if batch_id: + dbcon.update_many( + {"batch_id": batch_id, "status": "sent_for_reprocessing"}, + { + "$set": + { + "finish_date": datetime.now(), + "status": "republish_finished", + } + } + ) + dbcon.update_one( {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": "finished_ok", - "progress": 100, - "log": os.linesep.join(log_lines) - }} + { + "$set": + { + "finish_date": datetime.now(), + "status": "finished_ok", + "progress": 100, + "log": os.linesep.join(log_lines) + } + } ) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index c9612d89152..8d0eb773a22 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -314,7 +314,7 @@ def remotepublish(project, batch_path, user_email, targets=None): dbcon = get_webpublish_conn() _id = start_webpublish_log(dbcon, batch_id, user_email) - publish_and_log(dbcon, _id, log) + publish_and_log(dbcon, _id, log, batch_id=batch_id) log.info("Publish finished.") From 69a90001d347643a1dd207a77cd3b5b7ec78d27f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 19 Jan 2022 19:07:25 +0100 Subject: [PATCH 10/11] OP-2427 - refactor - status as code --- .../plugins/publish/collect_batch_data.py | 4 ++-- .../webserver_service/webpublish_routes.py | 10 +++++++--- .../webserver_service/webserver_cli.py | 13 +++++++++---- openpype/lib/remote_publish.py | 19 +++++++++++++------ openpype/pype_commands.py | 5 +++-- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py index a710fcb3e82..062c5ce0da5 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py @@ -12,7 +12,7 @@ parse_json, get_batch_asset_task_info ) -from openpype.lib.remote_publish import get_webpublish_conn +from openpype.lib.remote_publish import get_webpublish_conn, IN_PROGRESS_STATUS class CollectBatchData(pyblish.api.ContextPlugin): @@ -74,7 +74,7 @@ def _set_ctx_path(self, batch_data): dbcon.update_one( { "batch_id": batch_id, - "status": "in_progress" + "status": IN_PROGRESS_STATUS }, { "$set": { diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 099819f6cac..cafd6511670 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -11,10 +11,14 @@ from openpype.lib import OpenPypeMongoConnection from openpype_modules.avalon_apps.rest_api import _RestApiEndpoint -from openpype.lib.remote_publish import get_task_data from openpype.settings import get_project_settings from openpype.lib import PypeLogger +from openpype.lib.remote_publish import ( + get_task_data, + ERROR_STATUS, + REPROCESS_STATUS +) log = PypeLogger.get_logger("WebServer") @@ -371,12 +375,12 @@ class BatchReprocessEndpoint(_RestApiEndpoint): """Marks latest 'batch_id' for reprocessing, returns 404 if not found.""" async def post(self, batch_id) -> Response: batches = self.dbcon.find({"batch_id": batch_id, - "status": "error"}).sort("_id", -1) + "status": ERROR_STATUS}).sort("_id", -1) if batches: self.dbcon.update_one( {"_id": batches[0]["_id"]}, - {"$set": {"status": "reprocess"}} + {"$set": {"status": REPROCESS_STATUS}} ) output = [{"msg": "Batch id {} set to reprocess".format(batch_id)}] status = 200 diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 0d0a0223d90..909ea38bc69 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -20,6 +20,11 @@ TaskPublishEndpoint, UserReportEndpoint ) +from openpype.lib.remote_publish import ( + ERROR_STATUS, + REPROCESS_STATUS, + SENT_REPROCESSING_STATUS +) log = PypeLogger().get_logger("webserver_gui") @@ -125,7 +130,7 @@ def reprocess_failed(upload_dir, webserver_url): database_name = os.environ["OPENPYPE_DATABASE_NAME"] dbcon = mongo_client[database_name]["webpublishes"] - results = dbcon.find({"status": "reprocess"}) + results = dbcon.find({"status": REPROCESS_STATUS}) reprocessed_batches = set() for batch in results: if batch["batch_id"] in reprocessed_batches: @@ -143,7 +148,7 @@ def reprocess_failed(upload_dir, webserver_url): {"$set": { "finish_date": datetime.now(), - "status": "error", + "status": ERROR_STATUS, "progress": 100, "log": batch.get("log") + msg }} @@ -157,12 +162,12 @@ def reprocess_failed(upload_dir, webserver_url): dbcon.update_many( { "batch_id": batch["batch_id"], - "status": {"$in": ["error", "reprocess"]} + "status": {"$in": [ERROR_STATUS, REPROCESS_STATUS]} }, { "$set": { "finish_date": datetime.now(), - "status": "sent_for_reprocessing", + "status": SENT_REPROCESSING_STATUS, "progress": 100 } } diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index 181802792af..9632e63ea09 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -11,6 +11,13 @@ from openpype.lib.mongo import OpenPypeMongoConnection from openpype.lib.plugin_tools import parse_json +ERROR_STATUS = "error" +IN_PROGRESS_STATUS = "in_progress" +REPROCESS_STATUS = "reprocess" +SENT_REPROCESSING_STATUS = "sent_for_reprocessing" +FINISHED_REPROCESS_STATUS = "republishing_finished" +FINISHED_OK_STATUS = "finished_ok" + def headless_publish(log, close_plugin_name=None, is_test=False): """Runs publish in a opened host with a context and closes Python process. @@ -52,7 +59,7 @@ def start_webpublish_log(dbcon, batch_id, user): "batch_id": batch_id, "start_date": datetime.now(), "user": user, - "status": "in_progress", + "status": IN_PROGRESS_STATUS, "progress": 0 # integer 0-100, percentage }).inserted_id @@ -122,7 +129,7 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): {"$set": { "finish_date": datetime.now(), - "status": "error", + "status": ERROR_STATUS, "log": os.linesep.join(log_lines) }} @@ -146,12 +153,12 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): # final update if batch_id: dbcon.update_many( - {"batch_id": batch_id, "status": "sent_for_reprocessing"}, + {"batch_id": batch_id, "status": SENT_REPROCESSING_STATUS}, { "$set": { "finish_date": datetime.now(), - "status": "republish_finished", + "status": FINISHED_REPROCESS_STATUS, } } ) @@ -162,7 +169,7 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): "$set": { "finish_date": datetime.now(), - "status": "finished_ok", + "status": FINISHED_OK_STATUS, "progress": 100, "log": os.linesep.join(log_lines) } @@ -183,7 +190,7 @@ def fail_batch(_id, batches_in_progress, dbcon): {"$set": { "finish_date": datetime.now(), - "status": "error", + "status": ERROR_STATUS, "log": msg }} diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 8d0eb773a22..de0336be2b1 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -14,7 +14,8 @@ publish_and_log, fail_batch, find_variant_key, - get_task_data + get_task_data, + IN_PROGRESS_STATUS ) @@ -209,7 +210,7 @@ def remotepublishfromapp(project, batch_path, host_name, # safer to start logging here, launch might be broken altogether _id = start_webpublish_log(dbcon, batch_id, user_email) - batches_in_progress = list(dbcon.find({"status": "in_progress"})) + batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS})) if len(batches_in_progress) > 1: fail_batch(_id, batches_in_progress, dbcon) print("Another batch running, probably stuck, ask admin for help") From a6809a69aa6d44ca5c0ead628bda95f7753bf5bc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 20 Jan 2022 16:16:24 +0100 Subject: [PATCH 11/11] OP-2427 - conform to expected format for front end --- .../hosts/webpublisher/webserver_service/webpublish_routes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index cafd6511670..de098991041 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -308,7 +308,9 @@ async def get(self, batch_id) -> Response: if output: status = 200 else: - output = {"msg": "Batch id {} not found".format(batch_id)} + output = {"msg": "Batch id {} not found".format(batch_id), + "status": "queued", + "progress": 0} status = 404 body = self.resource.encode(output) return Response(