diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ee2bfff4..1b1f6a80 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.9.3" + ".": "0.9.4" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 08563c3d..fb743d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## [0.9.4](https://github.com/reanahub/reana-server/compare/0.9.3...0.9.4) (2024-11-29) + + +### Build + +* **python:** bump shared REANA packages as of 2024-11-28 ([#714](https://github.com/reanahub/reana-server/issues/714)) ([94fbf77](https://github.com/reanahub/reana-server/commit/94fbf7766218f4ffaf3f23be64ec6d46be1acb00)) + + +### Features + +* **config:** make ACCOUNTS_USERINFO_HEADERS customisable ([#713](https://github.com/reanahub/reana-server/issues/713)) ([8c01d51](https://github.com/reanahub/reana-server/commit/8c01d513c2365f337c26a2211c2ddb82df4186d4)) +* **config:** make APP_DEFAULT_SECURE_HEADERS customisable ([#713](https://github.com/reanahub/reana-server/issues/713)) ([1919358](https://github.com/reanahub/reana-server/commit/1919358cb3b05f09bceff9a904e9607760bc3fb1)) +* **config:** make PROXYFIX_CONFIG customisable ([#713](https://github.com/reanahub/reana-server/issues/713)) ([5b6c276](https://github.com/reanahub/reana-server/commit/5b6c276f57f642cc0965f096fa59875b9599df08)) +* **config:** support password-protected redis ([#713](https://github.com/reanahub/reana-server/issues/713)) ([a2aad8a](https://github.com/reanahub/reana-server/commit/a2aad8ac506b98e5c29d357cec65172b6437cc8f)) +* **ext:** improve error message for db decryption error ([#713](https://github.com/reanahub/reana-server/issues/713)) ([bbab1bf](https://github.com/reanahub/reana-server/commit/bbab1bf7338e9790e2195a02e320df16db1826f6)) + + +### Bug fixes + +* **config:** do not set DEBUG programmatically ([#713](https://github.com/reanahub/reana-server/issues/713)) ([c98cbc1](https://github.com/reanahub/reana-server/commit/c98cbc1d15afca9309e4839db543ac19cd2036ce)) +* **config:** read secret key from env ([#713](https://github.com/reanahub/reana-server/issues/713)) ([6ee6422](https://github.com/reanahub/reana-server/commit/6ee6422d87d38339b359ad7a306575b97f210440)) +* **get_workflow_specification:** avoid returning null parameters ([#689](https://github.com/reanahub/reana-server/issues/689)) ([46633d6](https://github.com/reanahub/reana-server/commit/46633d6bcc151c73880f9ecbd2c02d2246492794)) +* **reana-admin:** respect service domain when cleaning sessions ([#687](https://github.com/reanahub/reana-server/issues/687)) ([ede882d](https://github.com/reanahub/reana-server/commit/ede882d384ae0959eb8a9484b7d491baa628a1ee)) +* **set_workflow_status:** publish workflows to submission queue ([#691](https://github.com/reanahub/reana-server/issues/691)) ([6e35bd7](https://github.com/reanahub/reana-server/commit/6e35bd776e17c1bc04145c68c1f5ea3ce5143b7e)), closes [#690](https://github.com/reanahub/reana-server/issues/690) +* **start:** validate endpoint parameters ([#689](https://github.com/reanahub/reana-server/issues/689)) ([d2d3673](https://github.com/reanahub/reana-server/commit/d2d3673dac8917d746ddafd84bb3660e7f83c9b6)) + + +### Continuous integration + +* **commitlint:** improve checking of merge commits ([#689](https://github.com/reanahub/reana-server/issues/689)) ([69f45fc](https://github.com/reanahub/reana-server/commit/69f45fc3aae9bc625ed733de9af13eb7c0111048)) + ## [0.9.3](https://github.com/reanahub/reana-server/compare/0.9.2...0.9.3) (2024-03-04) diff --git a/Dockerfile b/Dockerfile index eec2c299..a9bfcca4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,7 +84,7 @@ CMD ["uwsgi --ini uwsgi.ini"] # Set image labels LABEL org.opencontainers.image.authors="team@reanahub.io" -LABEL org.opencontainers.image.created="2024-03-04" +LABEL org.opencontainers.image.created="2024-11-29" LABEL org.opencontainers.image.description="REANA reproducible analysis platform - server component" LABEL org.opencontainers.image.documentation="https://reana-server.readthedocs.io/" LABEL org.opencontainers.image.licenses="MIT" diff --git a/docs/openapi.json b/docs/openapi.json index b7a8ca89..e2121aed 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -4049,15 +4049,19 @@ "schema": { "properties": { "input_parameters": { + "description": "Optional. Additional input parameters that override the ones from the workflow specification.", "type": "object" }, "operational_options": { + "description": "Optional. Additional operational options for workflow execution.", "type": "object" }, "reana_specification": { + "description": "Optional. Replace the original workflow specification with the given one. Only considered when restarting a workflow.", "type": "object" }, "restart": { + "description": "Optional. If true, restart the given workflow.", "type": "boolean" } }, @@ -4446,6 +4450,11 @@ }, { "description": "Required. New workflow status.", + "enum": [ + "start", + "stop", + "deleted" + ], "in": "query", "name": "status", "required": true, @@ -4459,19 +4468,30 @@ "type": "string" }, { - "description": "Optional. Additional input parameters and operational options.", + "description": "Optional. Additional parameters to customise the workflow status change.", "in": "body", "name": "parameters", "required": false, "schema": { "properties": { - "CACHE": { - "type": "string" - }, "all_runs": { + "description": "Optional. If true, delete all runs of the workflow. Only allowed when status is `deleted`.", + "type": "boolean" + }, + "input_parameters": { + "description": "Optional. Additional input parameters that override the ones from the workflow specification. Only allowed when status is `start`.", + "type": "object" + }, + "operational_options": { + "description": "Optional. Additional operational options for workflow execution. Only allowed when status is `start`.", + "type": "object" + }, + "restart": { + "description": "Optional. If true, the workflow is a restart of an earlier workflow execution. Only allowed when status is `start`.", "type": "boolean" }, "workspace": { + "description": "Optional, but must be set to true if provided. If true, delete also the workspace of the workflow. Only allowed when status is `deleted`.", "type": "boolean" } }, diff --git a/reana_server/config.py b/reana_server/config.py index 6f774f34..0afed22c 100644 --- a/reana_server/config.py +++ b/reana_server/config.py @@ -167,8 +167,10 @@ def _(x): # Accounts # ======== #: Redis URL -ACCOUNTS_SESSION_REDIS_URL = "redis://{host}:6379/1".format( - host=REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES["cache"] +REANA_CACHE_PASSWORD = os.getenv("REANA_CACHE_PASSWORD", "") +ACCOUNTS_SESSION_REDIS_URL = "redis://:{password}@{host}:6379/1".format( + password=REANA_CACHE_PASSWORD, + host=REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES["cache"], ) #: Email address used as sender of account registration emails. SECURITY_EMAIL_SENDER = SUPPORT_EMAIL @@ -179,7 +181,9 @@ def _(x): #: and X-User-ID headers to HTTP response. You MUST ensure that NGINX (or other #: proxies) removes these headers again before sending the response to the #: client. Set to False, in case of doubt. -ACCOUNTS_USERINFO_HEADERS = True +ACCOUNTS_USERINFO_HEADERS = bool( + strtobool(os.getenv("ACCOUNTS_USERINFO_HEADERS", "False")) +) #: Disable password recovery by users. SECURITY_RECOVERABLE = False REANA_USER_EMAIL_CONFIRMATION = strtobool( @@ -217,7 +221,9 @@ def _(x): #: Secret key - each installation (dev, production, ...) needs a separate key. #: It should be changed before deploying. -SECRET_KEY = "CHANGE_ME" +SECRET_KEY = os.getenv("REANA_SECRET_KEY", "CHANGE_ME") +"""Secret key used for the application user sessions.""" + #: Sets cookie with the secure flag by default SESSION_COOKIE_SECURE = True #: Sets session to be samesite to avoid CSRF attacks @@ -234,8 +240,17 @@ def _(x): # Security configuration # ====================== -PROXYFIX_CONFIG = {"x_proto": 1} +PROXYFIX_CONFIG = json.loads(os.getenv("PROXYFIX_CONFIG", '{"x_proto": 1}')) + APP_DEFAULT_SECURE_HEADERS["content_security_policy"] = {} +APP_DEFAULT_SECURE_HEADERS.update( + json.loads(os.getenv("APP_DEFAULT_SECURE_HEADERS", "{}")) +) +if "REANA_FORCE_HTTPS" in os.environ: + APP_DEFAULT_SECURE_HEADERS["force_https"] = bool( + strtobool(os.getenv("REANA_FORCE_HTTPS")) + ) + APP_HEALTH_BLUEPRINT_ENABLED = False @@ -347,8 +362,6 @@ def _get_rate_limit(env_variable: str, default: str) -> str: OAUTHCLIENT_REMOTE_APPS["cern_openid"] = OAUTH_REMOTE_REST_APP OAUTHCLIENT_REST_REMOTE_APPS["cern_openid"] = OAUTH_REMOTE_REST_APP -DEBUG = True - SECURITY_PASSWORD_SALT = "security-password-salt" SECURITY_SEND_REGISTER_EMAIL = False diff --git a/reana_server/ext.py b/reana_server/ext.py index cce59b56..ce4b893d 100644 --- a/reana_server/ext.py +++ b/reana_server/ext.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of REANA. -# Copyright (C) 2019, 2020, 2021, 2022 CERN. +# Copyright (C) 2019, 2020, 2021, 2022, 2024 CERN. # # REANA is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -14,6 +14,7 @@ from flask_limiter.errors import RateLimitExceeded from marshmallow.exceptions import ValidationError from reana_commons.config import REANA_LOG_FORMAT, REANA_LOG_LEVEL +from sqlalchemy_utils.types.encrypted.padding import InvalidPaddingError from werkzeug.exceptions import UnprocessableEntity from invenio_oauthclient.signals import account_info_received @@ -60,6 +61,17 @@ def handle_args_validation_error(error: UnprocessableEntity): return jsonify({"message": error_message}), 400 +def handle_invalid_padding_error(error: InvalidPaddingError): + """Error handler for sqlalchemy_utils exception ``InvalidPaddingError``. + + This error handler raises an exception with a more understandable message. + """ + raise InvalidPaddingError( + "Error decrypting the database. Did you set the correct secret key? " + "If you changed the secret key, did you run the migration command?" + ) from error + + class REANA(object): """REANA Invenio app. @@ -106,3 +118,4 @@ def init_error_handlers(self, app): """Initialize custom error handlers.""" app.register_error_handler(RateLimitExceeded, handle_rate_limit_error) app.register_error_handler(UnprocessableEntity, handle_args_validation_error) + app.register_error_handler(InvalidPaddingError, handle_invalid_padding_error) diff --git a/reana_server/factory.py b/reana_server/factory.py index 0c3a7855..64e7dd8a 100644 --- a/reana_server/factory.py +++ b/reana_server/factory.py @@ -45,7 +45,6 @@ def create_minimal_app(config_mapping=None): app.config.from_object("reana_server.config") if config_mapping: app.config.from_mapping(config_mapping) - app.secret_key = "hyper secret key" app.session = Session diff --git a/reana_server/reana_admin/cli.py b/reana_server/reana_admin/cli.py index 0fbf07ad..6462fa44 100644 --- a/reana_server/reana_admin/cli.py +++ b/reana_server/reana_admin/cli.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of REANA. -# Copyright (C) 2020, 2021, 2022 CERN. +# Copyright (C) 2020, 2021, 2022, 2024 CERN. # # REANA is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -1089,7 +1089,7 @@ def interactive_session_cleanup( try: session_status = requests.get( - f"http://reana-run-session-{workflow_id}.{REANA_RUNTIME_KUBERNETES_NAMESPACE}.svc.cluster.local:8081/{workflow_id}/api/status", + f"http://reana-run-session-{workflow_id}.{REANA_RUNTIME_KUBERNETES_NAMESPACE}:8081/{workflow_id}/api/status", headers={"Authorization": f"token {token}"}, ).json() except Exception as e: diff --git a/reana_server/rest/workflows.py b/reana_server/rest/workflows.py index 475ff47c..880f674e 100644 --- a/reana_server/rest/workflows.py +++ b/reana_server/rest/workflows.py @@ -803,7 +803,8 @@ def get_workflow_specification(workflow_id_or_name, user): # noqa jsonify( { "specification": workflow.reana_specification, - "parameters": workflow.input_parameters, + # `input_parameters` can be null, if so return an empty dict + "parameters": workflow.input_parameters or {}, } ), 200, @@ -1171,10 +1172,87 @@ def get_workflow_status(workflow_id_or_name, user): # noqa return jsonify({"message": str(e)}), 500 +def _start_workflow(workflow_id_or_name, user, **parameters): + """Start given workflow by publishing it to the submission queue. + + This function is used by both the `set_workflow_status` and `start_workflow`. + """ + operational_options = parameters.get("operational_options", {}) + input_parameters = parameters.get("input_parameters", {}) + restart = parameters.get("restart", False) + reana_specification = parameters.get("reana_specification") + + try: + if not workflow_id_or_name: + raise ValueError("workflow_id_or_name is not supplied") + + workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, str(user.id_)) + operational_options = validate_operational_options( + workflow.type_, operational_options + ) + + restart_type = None + if restart: + if workflow.status not in [RunStatus.finished, RunStatus.failed]: + raise ValueError("Only finished or failed workflows can be restarted.") + if workflow.workspace_has_pending_retention_rules(): + raise ValueError( + "The workflow cannot be restarted because some retention rules are " + "currently being applied to the workspace. Please retry later." + ) + if reana_specification: + restart_type = reana_specification.get("workflow", {}).get("type", None) + workflow = clone_workflow(workflow, reana_specification, restart_type) + elif workflow.status != RunStatus.created: + raise ValueError( + "Workflow {} is already {} and cannot be started " + "again.".format(workflow.get_full_workflow_name(), workflow.status.name) + ) + if "yadage" in (workflow.type_, restart_type): + _load_and_save_yadage_spec(workflow, operational_options) + + validate_workflow( + workflow.reana_specification, input_parameters=input_parameters + ) + + # when starting the workflow, the scheduler will call RWC's `set_workflow_status` + # with the given `parameters` + publish_workflow_submission(workflow, user.id_, parameters) + response = { + "message": "Workflow submitted.", + "workflow_id": workflow.id_, + "workflow_name": workflow.name, + "status": RunStatus.queued.name, + "run_number": workflow.run_number, + "user": str(user.id_), + } + return response, 200 + except HTTPError as e: + logging.error(traceback.format_exc()) + return e.response.json(), e.response.status_code + except (REANAValidationError, ValidationError) as e: + logging.error(traceback.format_exc()) + return {"message": str(e)}, 400 + except ValueError as e: + logging.error(traceback.format_exc()) + return {"message": str(e)}, 403 + except Exception as e: + logging.error(traceback.format_exc()) + return {"message": str(e)}, 500 + + @blueprint.route("/workflows//start", methods=["POST"]) @signin_required() +@use_kwargs( + { + "operational_options": fields.Dict(location="json"), + "input_parameters": fields.Dict(location="json"), + "restart": fields.Boolean(location="json"), + "reana_specification": fields.Raw(location="json"), + } +) @check_quota -def start_workflow(workflow_id_or_name, user): # noqa +def start_workflow(workflow_id_or_name, user, **parameters): # noqa r"""Start workflow. --- post: @@ -1207,12 +1285,20 @@ def start_workflow(workflow_id_or_name, user): # noqa type: object properties: operational_options: - type: object - reana_specification: + description: Optional. Additional operational options for workflow execution. type: object input_parameters: + description: >- + Optional. Additional input parameters that override the ones from + the workflow specification. + type: object + reana_specification: + description: >- + Optional. Replace the original workflow specification with the given one. + Only considered when restarting a workflow. type: object restart: + description: Optional. If true, restart the given workflow. type: boolean responses: 200: @@ -1326,72 +1412,25 @@ def start_workflow(workflow_id_or_name, user): # noqa "message": "Status resume is not supported yet." } """ - try: - if not workflow_id_or_name: - raise ValueError("workflow_id_or_name is not supplied") - parameters = request.json if request.is_json else {} - workflow = _get_workflow_with_uuid_or_name(workflow_id_or_name, str(user.id_)) - operational_options = parameters.get("operational_options", {}) - operational_options = validate_operational_options( - workflow.type_, operational_options - ) - restart_type = None - if "restart" in parameters: - if workflow.status not in [RunStatus.finished, RunStatus.failed]: - raise ValueError("Only finished or failed workflows can be restarted.") - if workflow.workspace_has_pending_retention_rules(): - raise ValueError( - "The workflow cannot be restarted because some retention rules are " - "currently being applied to the workspace. Please retry later." - ) - restart_type = ( - parameters.get("reana_specification", {}) - .get("workflow", {}) - .get("type", None) - ) - workflow = clone_workflow( - workflow, parameters.get("reana_specification", None), restart_type - ) - elif workflow.status != RunStatus.created: - raise ValueError( - "Workflow {} is already {} and cannot be started " - "again.".format(workflow.get_full_workflow_name(), workflow.status.name) - ) - if "yadage" in (workflow.type_, restart_type): - _load_and_save_yadage_spec(workflow, operational_options) - - input_parameters = parameters.get("input_parameters", {}) - validate_workflow( - workflow.reana_specification, input_parameters=input_parameters - ) - - publish_workflow_submission(workflow, user.id_, parameters) - response = { - "message": "Workflow submitted.", - "workflow_id": workflow.id_, - "workflow_name": workflow.name, - "status": RunStatus.queued.name, - "run_number": workflow.run_number, - "user": str(user.id_), - } - return jsonify(response), 200 - except HTTPError as e: - logging.error(traceback.format_exc()) - return jsonify(e.response.json()), e.response.status_code - except (REANAValidationError, ValidationError) as e: - logging.error(traceback.format_exc()) - return jsonify({"message": str(e)}), 400 - except ValueError as e: - logging.error(traceback.format_exc()) - return jsonify({"message": str(e)}), 403 - except Exception as e: - logging.error(traceback.format_exc()) - return jsonify({"message": str(e)}), 500 + response, status_code = _start_workflow(workflow_id_or_name, user, **parameters) + return jsonify(response), status_code @blueprint.route("/workflows//status", methods=["PUT"]) @signin_required() -def set_workflow_status(workflow_id_or_name, user): # noqa +@use_kwargs( + { + "status": fields.Str(required=True, location="query"), + # parameters for "start" + "input_parameters": fields.Dict(location="json"), + "operational_options": fields.Dict(location="json"), + "restart": fields.Boolean(location="json"), + # parameters for "deleted" + "all_runs": fields.Boolean(location="json"), + "workspace": fields.Boolean(location="json"), + } +) +def set_workflow_status(workflow_id_or_name, user, status, **parameters): # noqa r"""Set workflow status. --- put: @@ -1415,6 +1454,10 @@ def set_workflow_status(workflow_id_or_name, user): # noqa description: Required. New workflow status. required: true type: string + enum: + - start + - stop + - deleted - name: access_token in: query description: The API access_token of workflow owner. @@ -1423,18 +1466,37 @@ def set_workflow_status(workflow_id_or_name, user): # noqa - name: parameters in: body description: >- - Optional. Additional input parameters and operational options. + Optional. Additional parameters to customise the workflow status change. required: false schema: type: object properties: - CACHE: - type: string + operational_options: + description: >- + Optional. Additional operational options for workflow execution. + Only allowed when status is `start`. + type: object + input_parameters: + description: >- + Optional. Additional input parameters that override the ones + from the workflow specification. Only allowed when status is `start`. + type: object + restart: + description: >- + Optional. If true, the workflow is a restart of an earlier workflow execution. + Only allowed when status is `start`. + type: boolean all_runs: + description: >- + Optional. If true, delete all runs of the workflow. + Only allowed when status is `deleted`. type: boolean workspace: + description: >- + Optional, but must be set to true if provided. + If true, delete also the workspace of the workflow. + Only allowed when status is `deleted`. type: boolean - responses: 200: description: >- @@ -1550,7 +1612,20 @@ def set_workflow_status(workflow_id_or_name, user): # noqa try: if not workflow_id_or_name: raise ValueError("workflow_id_or_name is not supplied") - status = request.args.get("status") + + if status == "start": + # We can't call directly RWC when starting a workflow, as otherwise + # the workflow would skip the queue. Instead, we do what the + # `start_workflow` endpoint does. + response, status_code = _start_workflow( + workflow_id_or_name, user, **parameters + ) + if "run_number" in response: + # run_number is returned by `start_workflow`, + # but not by `set_status_workflow` + del response["run_number"] + return jsonify(response), status_code + parameters = request.json if request.is_json else None response, http_response = current_rwc_api_client.api.set_workflow_status( user=str(user.id_), diff --git a/reana_server/version.py b/reana_server/version.py index c6fc8a0d..47c22f37 100644 --- a/reana_server/version.py +++ b/reana_server/version.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of REANA. -# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023 CERN. +# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024 CERN. # # REANA is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. diff --git a/tests/test_views.py b/tests/test_views.py index e974c50f..15873c66 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -208,7 +208,7 @@ def test_restart_workflow_validates_specification( workflow_specification["workflow"]["type"] = "unknown" body = { "reana_specification": workflow_specification, - "restart": "can be anything here doesnt matter", + "restart": True, } res = client.post( url_for("workflows.start_workflow", workflow_id_or_name="test"), @@ -320,8 +320,8 @@ def test_get_workflow_status(app, user0, _get_user_mock): assert res.status_code == 200 -def test_set_workflow_status(app, user0, _get_user_mock): - """Test get_workflow_logs view.""" +def test_set_workflow_status(app, default_user, _get_user_mock): + """Test set_workflow_status view.""" with app.test_client() as client: with patch( "reana_server.rest.workflows.current_rwc_api_client", @@ -343,7 +343,7 @@ def test_set_workflow_status(app, user0, _get_user_mock): headers={"Content-Type": "application/json"}, query_string={"access_token": user0.access_token}, ) - assert res.status_code == 500 + assert res.status_code == 422 res = client.put( url_for("workflows.set_workflow_status", workflow_id_or_name="1"),