renamed and refactored names for mscolab (#2499)

removed the xmlsec binary selector based on CONDA_PREFIX
provided separate config for msidp
separate dir for mscolab server data and work asynchronously by msolab ui


---------

Co-authored-by: Matthias Riße <9308656+matrss@users.noreply.github.com>

conftest.py

@@ -111,9 +111,9 @@ def generate_initial_config():
     root_fs.makedir('colabTestData')
 BASE_DIR = ROOT_DIR
 DATA_DIR = fs.path.join(ROOT_DIR, 'colabTestData')
-# mscolab data directory
-MSCOLAB_DATA_DIR = fs.path.join(DATA_DIR, 'filedata')
-MSCOLAB_SSO_DIR = fs.path.join(DATA_DIR, 'datasso')
+# mscolab data directory for operation git repositories
+OPERATIONS_DATA = fs.path.join(DATA_DIR, 'filedata')
+SSO_DIR = fs.path.join(DATA_DIR, 'datasso')
 
 # In the unit days when Operations get archived because not used
 ARCHIVE_THRESHOLD = 30

docs/conf_sso_test_msscolab.rst

@@ -30,7 +30,13 @@ To set up a local identity provider with the mscolab server, you'll first need t
 ----------------
 Before getting started, you should correctly activate the environments, set the correct Python path as explained in the mss instructions : https://github.com/Open-MSS/MSS/tree/develop#readme
 
+Setup your
 
+**msidp_settings.py**
+
+ .. literalinclude:: samples/config/msidp/msidp_settings.py.sample
+
+.. _configuration-mscolab:
 
 2. Generate Keys, Certificates, and backend_saml files
 ------------------------------------------------------

docs/samples/config/mscolab/mscolab_settings.py.sample

@@ -44,8 +44,16 @@ BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 # Directory in which all data related to Mscolab is stored
 DATA_DIR = os.path.join(BASE_DIR, "colabdata")
 
-# Where mscolab project files are stored on the server
-MSCOLAB_DATA_DIR = os.path.join(DATA_DIR, 'filedata')
+# mscolab data directory for operation git repositories
+OPERATIONS_DATA = os.path.join(DATA_DIR, 'filedata')
+
+# SSO by SAML2 is optional
+
+# dir where mscolab single sign-on process files are stored
+SSO_DIR = os.path.join(DATA_DIR, 'datasso')
+
+# Enable SSL certificate verification during SSO between MSColab and IdP
+SSO_SSL_CERT_VERIFICATION = True
 
 # Directory where uploaded images and documents in the chat are stored
 UPLOAD_FOLDER = os.path.join(DATA_DIR, 'uploads')
@@ -99,9 +107,3 @@ USE_SAML2 = False
 # all users in that Group are set to the operations of that category
 # having the roles in the TexGroup
 GROUP_POSTFIX = "Group"
-
-# Enable SSL certificate verification during SSO between MSColab and IdP
-ENABLE_SSO_SSL_CERT_VERIFICATION = True
-
-# dir where mscolab single sign process files are stored
-MSCOLAB_SSO_DIR = os.path.join(DATA_DIR, 'datasso')

docs/samples/config/mscolab/setup_saml2_backend.py.sample

@@ -22,19 +22,19 @@ class setup_saml2_backend:
 
     ]
 
-    if os.path.exists(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml"):
-        with open(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj:
+    if os.path.exists(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml"):
+        with open(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj:
             yaml_data = yaml.safe_load(fobj)
         # go through configured IDPs and set conf file paths for particular files
         for configured_idp in CONFIGURED_IDPS:
             # set CRTs and metadata paths for the localhost_test_idp
             if 'localhost_test_idp' == configured_idp['idp_identity_name']:
                 yaml_data["config"]["localhost_test_idp"]["key_file"] = \
-                    f'{mscolab_settings.MSCOLAB_SSO_DIR}/key_mscolab.key'  # set path to your mscolab key file
+                    f'{mscolab_settings.SSO_DIR}/key_mscolab.key'  # set path to your mscolab key file
                 yaml_data["config"]["localhost_test_idp"]["cert_file"] = \
-                    f'{mscolab_settings.MSCOLAB_SSO_DIR}/crt_mscolab.crt'  # set path to your mscolab certificate file
+                    f'{mscolab_settings.SSO_DIR}/crt_mscolab.crt'  # set path to your mscolab certificate file
                 yaml_data["config"]["localhost_test_idp"]["metadata"]["local"][0] = \
-                    f'{mscolab_settings.MSCOLAB_SSO_DIR}/idp.xml'  # set path to your idp metadata xml file
+                    f'{mscolab_settings.SSO_DIR}/idp.xml'  # set path to your idp metadata xml file
 
                 # configuration localhost_test_idp Saml2Client
                 try:

docs/samples/config/msidp/msidp_settings.py.sample

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""
+
+    mslib.msidp.conf.py.example
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    config for msidp
+
+    This file is part of mss.
+
+    :copyright: Copyright 2023 Nilupul Manodya
+    :copyright: Copyright 2023-2024 by the MSS team, see AUTHORS.
+    :license: APACHE-2.0, see LICENSE for details.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+import os
+
+
+# our default dir for mss content
+BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss')
+
+DATA_DIR = os.path.join(BASE_DIR, "colabdata")
+
+# dir where mscolab single sign-on process files are stored
+SSO_DIR = os.path.join(DATA_DIR, 'datasso')

docs/sso_via_saml_mscolab.rst

@@ -471,7 +471,7 @@ Configuration in MSColab settings for Keycloak
 
 Configuration mss_saml2_backend.yaml file
 
-    Create your mss_saml2_backend.yaml file in your ``MSCOLAB_SSO_DIR``.
+    Create your mss_saml2_backend.yaml file in your ``SSO_DIR``.
 
         .. code:: text
 

mslib/mscolab/app/__init__.py

@@ -52,7 +52,7 @@
 APP.config.from_object(__name__)
 APP.route = prefix_route(APP.route, SCRIPT_NAME)
 
-APP.config['MSCOLAB_DATA_DIR'] = mscolab_settings.MSCOLAB_DATA_DIR
+APP.config['OPERATIONS_DATA'] = mscolab_settings.OPERATIONS_DATA
 APP.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI
 APP.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 APP.config['SQLALCHEMY_ECHO'] = mscolab_settings.SQLALCHEMY_ECHO

mslib/mscolab/conf.py

@@ -53,12 +53,20 @@ class default_mscolab_settings:
     CORS_ORIGINS = ["*"]
 
     # dir where msui output files are stored
-    BASE_DIR = os.path.expanduser("~")
+    BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss')
 
     DATA_DIR = os.path.join(BASE_DIR, "colabdata")
 
-    # mscolab data directory
-    MSCOLAB_DATA_DIR = os.path.join(DATA_DIR, 'filedata')
+    # mscolab data directory for operation git repositories
+    OPERATIONS_DATA = os.path.join(DATA_DIR, 'filedata')
+
+    # SSO by SAML2 is optional
+
+    # dir where mscolab single sign-on process files are stored
+    SSO_DIR = os.path.join(DATA_DIR, 'datasso')
+
+    # Enable SSL certificate verification during SSO between MSColab and IdP
+    SSO_SSL_CERT_VERIFICATION = True
 
     # MYSQL CONNECTION STRING: "mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>?charset=utf8mb4"
     SQLALCHEMY_DB_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'mscolab.db')
@@ -126,12 +134,6 @@ class default_mscolab_settings:
     # accounts on a database on the server
     DIRECT_LOGIN = True
 
-    # Enable SSL certificate verification during SSO between MSColab and IdP
-    ENABLE_SSO_SSL_CERT_VERIFICATION = True
-
-    # dir where mscolab single sign process files are stored
-    MSCOLAB_SSO_DIR = os.path.join(DATA_DIR, 'datasso')
-
 
 mscolab_settings = default_mscolab_settings()
 
@@ -165,19 +167,19 @@ class setup_saml2_backend:
             #     }
             # },
         ]
-        if os.path.exists(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml"):
-            with open(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj:
+        if os.path.exists(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml"):
+            with open(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj:
                 yaml_data = yaml.safe_load(fobj)
             # go through configured IDPs and set conf file paths for particular files
             for configured_idp in CONFIGURED_IDPS:
                 # set CRTs and metadata paths for the localhost_test_idp
                 if 'localhost_test_idp' == configured_idp['idp_identity_name']:
                     yaml_data["config"]["localhost_test_idp"]["key_file"] = \
-                        f'{mscolab_settings.MSCOLAB_SSO_DIR}/key_mscolab.key'
+                        f'{mscolab_settings.SSO_DIR}/key_mscolab.key'
                     yaml_data["config"]["localhost_test_idp"]["cert_file"] = \
-                        f'{mscolab_settings.MSCOLAB_SSO_DIR}/crt_mscolab.crt'
+                        f'{mscolab_settings.SSO_DIR}/crt_mscolab.crt'
                     yaml_data["config"]["localhost_test_idp"]["metadata"]["local"][0] = \
-                        f'{mscolab_settings.MSCOLAB_SSO_DIR}/idp.xml'
+                        f'{mscolab_settings.SSO_DIR}/idp.xml'
 
                     # configuration localhost_test_idp Saml2Client
                     try:
@@ -187,7 +189,7 @@ class setup_saml2_backend:
                                            Ignore this warning when you initialize metadata.")
 
                         localhost_test_idp = SPConfig().load(yaml_data["config"]["localhost_test_idp"])
-                        localhost_test_idp.verify_ssl_cert = mscolab_settings.ENABLE_SSO_SSL_CERT_VERIFICATION
+                        localhost_test_idp.verify_ssl_cert = mscolab_settings.SSO_SSL_CERT_VERIFICATION
                         sp_localhost_test_idp = Saml2Client(localhost_test_idp)
 
                         configured_idp['idp_data']['saml2client'] = sp_localhost_test_idp

mslib/mscolab/mscolab.py

@@ -102,9 +102,9 @@ def handle_mscolab_certificate_init():
 
     try:
         cmd = ["openssl", "req", "-newkey", "rsa:4096", "-keyout",
-               os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "key_mscolab.key"),
+               os.path.join(mscolab_settings.SSO_DIR, "key_mscolab.key"),
                "-nodes", "-x509", "-days", "365", "-batch", "-subj",
-               "/CN=localhost", "-out", os.path.join(mscolab_settings.MSCOLAB_SSO_DIR,
+               "/CN=localhost", "-out", os.path.join(mscolab_settings.SSO_DIR,
                                                      "crt_mscolab.crt")]
         subprocess.run(cmd, check=True)
         logging.info("generated CRTs for the mscolab server.")
@@ -119,9 +119,9 @@ def handle_local_idp_certificate_init():
 
     try:
         cmd = ["openssl", "req", "-newkey", "rsa:4096", "-keyout",
-               os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "key_local_idp.key"),
+               os.path.join(mscolab_settings.SSO_DIR, "key_local_idp.key"),
                "-nodes", "-x509", "-days", "365", "-batch", "-subj",
-               "/CN=localhost", "-out", os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "crt_local_idp.crt")]
+               "/CN=localhost", "-out", os.path.join(mscolab_settings.SSO_DIR, "crt_local_idp.crt")]
         subprocess.run(cmd, check=True)
         logging.info("generated CRTs for the local identity provider")
         return True
@@ -252,7 +252,7 @@ def handle_mscolab_backend_yaml_init():
   #       name_id_format_allow_create: true
 """
     try:
-        file_path = os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "mss_saml2_backend.yaml")
+        file_path = os.path.join(mscolab_settings.SSO_DIR, "mss_saml2_backend.yaml")
         with open(file_path, "w", encoding="utf-8") as file:
             file.write(saml_2_backend_yaml_content)
         return True
@@ -278,7 +278,7 @@ def handle_mscolab_metadata_init(repo_exists):
         process = subprocess.Popen(command)
         cmd_curl = ["curl", "--retry", "5", "--retry-connrefused", "--retry-delay", "3",
                     "http://localhost:8083/metadata/localhost_test_idp",
-                    "-o", os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "metadata_sp.xml")]
+                    "-o", os.path.join(mscolab_settings.SSO_DIR, "metadata_sp.xml")]
         subprocess.run(cmd_curl, check=True)
         process.terminate()
         logging.info('mscolab metadata file generated succesfully')
@@ -293,8 +293,8 @@ def handle_local_idp_metadata_init(repo_exists):
     print('generating metadata for localhost identity provider')
 
     try:
-        if os.path.exists(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml")):
-            os.remove(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml"))
+        if os.path.exists(os.path.join(mscolab_settings.SSO_DIR, "idp.xml")):
+            os.remove(os.path.join(mscolab_settings.SSO_DIR, "idp.xml"))
 
         idp_conf_path = os.path.join("mslib", "msidp", "idp_conf.py")
 
@@ -305,15 +305,15 @@ def handle_local_idp_metadata_init(repo_exists):
 
         cmd = ["make_metadata", idp_conf_path]
 
-        with open(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml"),
+        with open(os.path.join(mscolab_settings.SSO_DIR, "idp.xml"),
                   "w", encoding="utf-8") as output_file:
             subprocess.run(cmd, stdout=output_file, check=True)
         logging.info("idp metadata file generated successfully")
         return True
     except subprocess.CalledProcessError as error:
         # Delete the idp.xml file when the subprocess fails
-        if os.path.exists(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml")):
-            os.remove(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml"))
+        if os.path.exists(os.path.join(mscolab_settings.SSO_DIR, "idp.xml")):
+            os.remove(os.path.join(mscolab_settings.SSO_DIR, "idp.xml"))
         print(f"Error while generating metadata for localhost identity provider: {error}")
         return False
 
@@ -323,8 +323,8 @@ def handle_sso_crts_init():
         This will generate necessary CRTs files for sso in mscolab through localhost idp
     """
     print("\n\nmscolab sso conf initiating......")
-    if os.path.exists(mscolab_settings.MSCOLAB_SSO_DIR):
-        shutil.rmtree(mscolab_settings.MSCOLAB_SSO_DIR)
+    if os.path.exists(mscolab_settings.SSO_DIR):
+        shutil.rmtree(mscolab_settings.SSO_DIR)
     create_files()
     if not handle_mscolab_certificate_init():
         print('Error while handling mscolab certificate.')

mslib/mscolab/seed.py

@@ -62,7 +62,7 @@ def add_all_users_default_operation(path='TEMPLATE', description="Operation to k
             operation = Operation(path, description)
             db.session.add(operation)
             db.session.commit()
-            with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir:
+            with fs.open_fs(mscolab_settings.OPERATIONS_DATA) as file_dir:
                 if not file_dir.exists(path):
                     file_dir.makedir(path)
                     file_dir.writetext(f'{path}/main.ftml', mscolab_settings.STUB_CODE)
@@ -150,7 +150,7 @@ def add_operation(operation_name, description):
             operation = Operation(operation_name, description)
             db.session.add(operation)
             db.session.commit()
-            with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir:
+            with fs.open_fs(mscolab_settings.OPERATIONS_DATA) as file_dir:
                 if not file_dir.exists(operation_name):
                     file_dir.makedir(operation_name)
                     file_dir.writetext(f'{operation_name}/main.ftml', mscolab_settings.STUB_CODE)
@@ -396,7 +396,7 @@ def seed_data():
     db.session.commit()
     db.session.close()
 
-    with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir:
+    with fs.open_fs(mscolab_settings.OPERATIONS_DATA) as file_dir:
         file_paths = ['one', 'two', 'three', 'four', 'Admin_Test', 'test_mscolab']
         for file_path in file_paths:
             file_dir.makedir(file_path)

mslib/mscolab/sockets_manager.py

@@ -334,7 +334,7 @@ def _setup_managers(app):
     """
 
     cm = ChatManager()
-    fm = FileManager(app.config["MSCOLAB_DATA_DIR"])
+    fm = FileManager(app.config["OPERATIONS_DATA"])
     sm = SocketsManager(cm, fm)
     # sockets related handlers
     socketio.on_event('connect', sm.handle_connect)

mslib/mscolab/utils.py

@@ -82,6 +82,6 @@ def os_fs_create_dir(directory_path):
 
 
 def create_files():
-    os_fs_create_dir(mscolab_settings.MSCOLAB_DATA_DIR)
+    os_fs_create_dir(mscolab_settings.OPERATIONS_DATA)
     os_fs_create_dir(mscolab_settings.UPLOAD_FOLDER)
-    os_fs_create_dir(mscolab_settings.MSCOLAB_SSO_DIR)
+    os_fs_create_dir(mscolab_settings.SSO_DIR)

mslib/msidp/conf.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+
+    mslib.msidp.conf
+    ~~~~~~~~~~~~~~~~
+
+    config for msidp.
+
+    This file is part of MSS.
+
+    :copyright: Copyright 2023 Nilupul Manodya
+    :copyright: Copyright 2023-2024 by the MSS team, see AUTHORS.
+    :license: APACHE-2.0, see LICENSE for details.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+import os
+import logging
+
+
+class default_msidp_settings:
+    # our default dir for mss content
+    BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss')
+
+    DATA_DIR = os.path.join(BASE_DIR, "colabdata")
+
+    # dir where mscolab single sign-on process files are stored
+    SSO_DIR = os.path.join(DATA_DIR, 'datasso')
+
+
+msidp_settings = default_msidp_settings()
+
+try:
+    import msidp_settings as user_settings
+    logging.info("Using user defined settings")
+    msidp_settings.__dict__.update(user_settings.__dict__)
+except ImportError as ex:
+    logging.warning(u"Couldn't import msidp_settings (ImportError:'%s'), using dummy config.", ex)