COMMIT merge

Open-MSS · Sep 26, 2024 · 7c45955 · 7c45955
2 parents e8c1f49 + 22aa853
commit 7c45955
Show file tree

Hide file tree

Showing 97 changed files with 2,403 additions and 3,066 deletions.
diff --git a/.github/workflows/testing-all-oses.yml b/.github/workflows/testing-all-oses.yml
@@ -48,6 +48,4 @@ jobs:
       # I have no idea yet on why this happens and how to fix it.
       # Even a module level skip is not enough, they need to be completely ignored.
       # TODO: fix those tests and drop the ignores
-      run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib
-        --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py
-        tests
+      run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib tests
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,6 +1,16 @@
 Changelog
 =========
 
+Version 9.2.0
+~~~~~~~~~~~~~
+
+Bug fix release and minor enhancements:
+We added a verification for xml data and changed the startup of the SocketsManager.
+
+All changes:
+https://github.com/Open-MSS/MSS/milestone/106?closed=1
+
+
 Version 9.1.0
 ~~~~~~~~~~~~~
 

diff --git a/README.md b/README.md
@@ -1,3 +1,8 @@
+**Chat:**
+[![IRC: #mss-general on libera.chat](https://img.shields.io/badge/libera.chat-%23MSS_General-blue)](https://web.libera.chat/?channels=#mss-general)
+[![IRC: #mss-gsoc on libera.chat](https://img.shields.io/badge/libera.chat-%23MSS_GSoC-brightgreen)](https://web.libera.chat/?channels=#mss-gsoc)
+
+
 Mission Support System Usage Guidelines
 =======================================
 

diff --git a/conftest.py b/conftest.py
@@ -111,9 +111,9 @@ def generate_initial_config():
     root_fs.makedir('colabTestData')
 BASE_DIR = ROOT_DIR
 DATA_DIR = fs.path.join(ROOT_DIR, 'colabTestData')
-# mscolab data directory
-MSCOLAB_DATA_DIR = fs.path.join(DATA_DIR, 'filedata')
-MSCOLAB_SSO_DIR = fs.path.join(DATA_DIR, 'datasso')
+# mscolab data directory for operation git repositories
+OPERATIONS_DATA = fs.path.join(DATA_DIR, 'filedata')
+SSO_DIR = fs.path.join(DATA_DIR, 'datasso')
 
 # In the unit days when Operations get archived because not used
 ARCHIVE_THRESHOLD = 30

diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html
@@ -0,0 +1,21 @@
+{% extends '!footer.html' %}
+
+{% block extrafooter %}
+    <HR>
+    <div class="footer-extra">
+        <p>
+            <a class="reference external" href="https://anaconda.org/conda-forge/mss"><img alt="Conda Version" src="https://img.shields.io/conda/vn/conda-forge/mss.svg" /></a>
+            <a class="reference external" href="https://doi.org/10.5281/zenodo.6572620"><img alt="DOI" src="https://zenodo.org/badge/DOI/10.5281/zenodo.6572620.svg" /></a>
+            <a class="reference external" href="https://www.fz-juelich.de/en/rse/jurse-community/jurse-code-of-the-month/july-2024"><img alt="JuRSE Code Pick" src="https://img.shields.io/badge/JuRSE_Code_Pick-July_2024-blue" /></a>
+            <a class="reference external" href="https://github.com/Open-MSS/MSS/issues"><img alt="GitHub Issues" src="https://img.shields.io/github/issues/Open-MSS/MSS" /></a>
+            <a class="reference external" href="https://coveralls.io/github/Open-MSS/MSS?branch=develop"><img alt="Coverage Status" src="https://coveralls.io/repos/github/Open-MSS/MSS/badge.svg?branch=develop" /></a>
+            <a class="reference external" href="https://anaconda.org/conda-forge/mss"><img alt="Conda Platforms" src="https://img.shields.io/conda/pn/conda-forge/mss.svg" /></a>
+            <a class="reference external" href="https://anaconda.org/conda-forge/mss"><img alt="Conda Recipe" src="https://img.shields.io/badge/recipe-mss-green.svg" /></a>
+            <a class="reference external" href="https://anaconda.org/conda-forge/mss"><img alt="Conda Downloads" src="https://img.shields.io/conda/dn/conda-forge/mss.svg" /></a>
+        </p>
+        <p>
+            <a href="https://web.libera.chat/?channels=#mss-general" target="_blank" rel="noopener"><img src="https://img.shields.io/badge/libera.chat-%23MSS_General-blue" alt="IRC: #mss-general on libera.chat"></a>
+            <a href="https://web.libera.chat/?channels=#mss-gsoc" target="_blank" rel="noopener"><img src="https://img.shields.io/badge/libera.chat-%23MSS_GSoC-brightgreen" alt="IRC: #mss-gsoc on libera.chat"></a>
+        </p>
+  </div>
+{% endblock %}
diff --git a/docs/conf.py b/docs/conf.py
@@ -226,7 +226,7 @@ def get_tutorial_images():
     html_style = 'css/mss.css'
 else:
     htmls_static_path = ['_static']
-    html_css_files = ['mss.css']
+    html_css_files = ['css/mss.css']
     html_context = {
         'display_github': False,  # Add 'Edit on Bitbucket' link instead of 'View page source'
         'last_updated': True,

diff --git a/docs/conf_sso_test_msscolab.rst b/docs/conf_sso_test_msscolab.rst
@@ -30,7 +30,13 @@ To set up a local identity provider with the mscolab server, you'll first need t
 ----------------
 Before getting started, you should correctly activate the environments, set the correct Python path as explained in the mss instructions : https://github.com/Open-MSS/MSS/tree/develop#readme
 
+Setup your
 
+**msidp_settings.py**
+
+ .. literalinclude:: samples/config/msidp/msidp_settings.py.sample
+
+.. _configuration-mscolab:
 
 2. Generate Keys, Certificates, and backend_saml files
 ------------------------------------------------------

diff --git a/docs/environment.yml b/docs/environment.yml
@@ -15,7 +15,7 @@ dependencies:
       - xstatic
       - defusedxml
       - sphinx_rtd_theme
-      - sphinxcontrib-video
+      - sphinxcontrib-video>=0.2.1
       - sphinx
       - fs
       - netCDF4

diff --git a/docs/installation.rst b/docs/installation.rst
@@ -124,23 +124,9 @@ versions for dependencies. ::
     (mssenv) $ msui
 
 
-Update
-------
-
-Builtin Update
+Update Methods
 ..............
 
-Since version 5.0 we provide a feature for updating MSS by the UI or the command line
-After you started the MSS UI it informs you after a while if there is a new update available.
-From the command line you can trigger this update feature by ::
-
-    (mssenv) $ msui --update
-
-
-
-Other Methods
-.............
-
 For updating an existing MSS installation to the current version, it is best to install
 it into a new environment. If your current version is not far behind the new version
 you could try the `mamba update mss` as described.

diff --git a/docs/mscolab.rst b/docs/mscolab.rst
@@ -190,6 +190,8 @@ Operation based features
   - All the operations the user has created or has been added to can be found in Mscolab's main window along with the user's access level.
   - To start working on an operation the user needs to select it which enables all the operation related buttons.
   - Any change made to an operation by a user will be shared with everyone in real-time unless `Work Locally` is turned on.(More on this later)
+  - Operations can be manually archived to remove them from normal view without having to delete them.
+    They can be found again in the Archive view, where they can be unarchived for further use.
 
 Operation Permissions
 .....................

diff --git a/docs/mss_theme/css/mss.css b/docs/mss_theme/css/mss.css
@@ -14,3 +14,7 @@
   margin-bottom: auto;
   margin-left: auto;
 }
+
+video {
+  max-width: 100%;
+}
diff --git a/docs/samples/automation/retriever.py b/docs/samples/automation/retriever.py
@@ -31,6 +31,7 @@
 import io
 import os
 import xml
+import defusedxml.minidom
 import requests
 from fs import open_fs
 import PIL.Image
@@ -58,7 +59,7 @@ def load_from_ftml(filename):
     _fs = open_fs(_dirname)
     datasource = _fs.open(_name)
     try:
-        doc = xml.dom.minidom.parse(datasource)
+        doc = defusedxml.minidom.parse(datasource)
     except xml.parsers.expat.ExpatError as ex:
         raise SyntaxError(str(ex))
 

diff --git a/docs/samples/config/mscolab/mscolab_settings.py.sample b/docs/samples/config/mscolab/mscolab_settings.py.sample
@@ -44,8 +44,16 @@ BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 # Directory in which all data related to Mscolab is stored
 DATA_DIR = os.path.join(BASE_DIR, "colabdata")
 
-# Where mscolab project files are stored on the server
-MSCOLAB_DATA_DIR = os.path.join(DATA_DIR, 'filedata')
+# mscolab data directory for operation git repositories
+OPERATIONS_DATA = os.path.join(DATA_DIR, 'filedata')
+
+# SSO by SAML2 is optional
+
+# dir where mscolab single sign-on process files are stored
+SSO_DIR = os.path.join(DATA_DIR, 'datasso')
+
+# Enable SSL certificate verification during SSO between MSColab and IdP
+SSO_SSL_CERT_VERIFICATION = True
 
 # Directory where uploaded images and documents in the chat are stored
 UPLOAD_FOLDER = os.path.join(DATA_DIR, 'uploads')
@@ -99,9 +107,3 @@ USE_SAML2 = False
 # all users in that Group are set to the operations of that category
 # having the roles in the TexGroup
 GROUP_POSTFIX = "Group"
-
-# Enable SSL certificate verification during SSO between MSColab and IdP
-ENABLE_SSO_SSL_CERT_VERIFICATION = True
-
-# dir where mscolab single sign process files are stored
-MSCOLAB_SSO_DIR = os.path.join(DATA_DIR, 'datasso')
diff --git a/docs/samples/config/mscolab/setup_saml2_backend.py.sample b/docs/samples/config/mscolab/setup_saml2_backend.py.sample
@@ -22,19 +22,19 @@ class setup_saml2_backend:
 
     ]
 
-    if os.path.exists(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml"):
-        with open(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj:
+    if os.path.exists(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml"):
+        with open(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj:
             yaml_data = yaml.safe_load(fobj)
         # go through configured IDPs and set conf file paths for particular files
         for configured_idp in CONFIGURED_IDPS:
             # set CRTs and metadata paths for the localhost_test_idp
             if 'localhost_test_idp' == configured_idp['idp_identity_name']:
                 yaml_data["config"]["localhost_test_idp"]["key_file"] = \
-                    f'{mscolab_settings.MSCOLAB_SSO_DIR}/key_mscolab.key'  # set path to your mscolab key file
+                    f'{mscolab_settings.SSO_DIR}/key_mscolab.key'  # set path to your mscolab key file
                 yaml_data["config"]["localhost_test_idp"]["cert_file"] = \
-                    f'{mscolab_settings.MSCOLAB_SSO_DIR}/crt_mscolab.crt'  # set path to your mscolab certificate file
+                    f'{mscolab_settings.SSO_DIR}/crt_mscolab.crt'  # set path to your mscolab certificate file
                 yaml_data["config"]["localhost_test_idp"]["metadata"]["local"][0] = \
-                    f'{mscolab_settings.MSCOLAB_SSO_DIR}/idp.xml'  # set path to your idp metadata xml file
+                    f'{mscolab_settings.SSO_DIR}/idp.xml'  # set path to your idp metadata xml file
 
                 # configuration localhost_test_idp Saml2Client
                 try:

diff --git a/docs/samples/config/msidp/msidp_settings.py.sample b/docs/samples/config/msidp/msidp_settings.py.sample
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""
+
+    mslib.msidp.conf.py.example
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    config for msidp
+
+    This file is part of mss.
+
+    :copyright: Copyright 2023 Nilupul Manodya
+    :copyright: Copyright 2023-2024 by the MSS team, see AUTHORS.
+    :license: APACHE-2.0, see LICENSE for details.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+import os
+
+
+# our default dir for mss content
+BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss')
+
+DATA_DIR = os.path.join(BASE_DIR, "colabdata")
+
+# dir where mscolab single sign-on process files are stored
+SSO_DIR = os.path.join(DATA_DIR, 'datasso')
diff --git a/docs/sso_via_saml_mscolab.rst b/docs/sso_via_saml_mscolab.rst
@@ -471,7 +471,7 @@ Configuration in MSColab settings for Keycloak
 
 Configuration mss_saml2_backend.yaml file
 
-    Create your mss_saml2_backend.yaml file in your ``MSCOLAB_SSO_DIR``.
+    Create your mss_saml2_backend.yaml file in your ``SSO_DIR``.
 
         .. code:: text
 

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
@@ -9,7 +9,7 @@ a `public share <https://fz-juelich.sciebo.de/s/IzNGi24Ki68iV7t#pdfviewer>`_
 Get familiar by some videos about the Mission Support System (MSS).
 
 
-  .. video:: _static/mp4/tutorial_waypoints.mp4
+  .. video:: videos/mp4/tutorial_waypoints.mp4
      :autoplay:
      :loop:
      :alt: This is the waypoints tutorial,i.e., whenever we are going to plan

diff --git a/docs/tutorials/tutorial_hexagoncontrol.rst b/docs/tutorials/tutorial_hexagoncontrol.rst
@@ -3,7 +3,7 @@ Table View and Hexagon Flight Patterns
 
 For tomographic imaging, a hexagonal flight pattern can be integrated by a docking widget of Table View
 
-   .. video:: ../_static/mp4/tutorial_hexagoncontrol.mp4
+   .. video:: ../videos/mp4/tutorial_hexagoncontrol.mp4
      :alt: Top View windows is opened (CTRL+H).
            We select the "global (cyl)" for the world map.
            Zooming in for the required region.

diff --git a/docs/tutorials/tutorial_introduction_topview.rst b/docs/tutorials/tutorial_introduction_topview.rst
@@ -3,7 +3,7 @@ Top View and Selecting of Layers
 
 Selection and display of different data in the Top View with the help of the layer chooser.
 
-  .. video:: ../_static/mp4/tutorial_wms.mp4
+  .. video:: ../videos/mp4/tutorial_wms.mp4
      :alt: When we open the Top View (CTRL+H) of the map, the Web Map Service is already opened by default.
            It collects its data from the server: "open-mss dot org" that provides demodata for the meteorological or
            atmospheric information as layer lists.

diff --git a/docs/tutorials/tutorial_kml.rst b/docs/tutorials/tutorial_kml.rst
@@ -3,7 +3,7 @@ Top View and KML Data
 
 es can be displayed in the Top View. Color and line width can be adjusted.
 
-  .. video:: ../_static/mp4/tutorial_kml.mp4
+  .. video:: ../videos/mp4/tutorial_kml.mp4
      :alt: Open the TopView (CTRL+H)
            After clicking on "(select to open control)", click on KML OVERLAY. The UI will look as shown.
            KML files can be used to show the geographical boundary which helps in planning the WAY POINTS.
@@ -23,4 +23,4 @@ es can be displayed in the Top View. Color and line width can be adjusted.
            Changing it for some time...
            Now, we change line width by changing its numerical value whose range is 0 to 10.
            We change it to different values to obtain different linewidths.
-           The top view is closed and the tutorial ends.
+           The top view is closed and the tutorial ends.
diff --git a/docs/tutorials/tutorial_mscolab.rst b/docs/tutorials/tutorial_mscolab.rst
@@ -5,7 +5,7 @@ Using the different views of the MSUI with a fictitious flight path and demo dat
 In comparison to the standalone mode of the MSUI an example setup of users is
 shown on a MSColab server and the possibilities of interactions.
 
-  .. video:: ../_static/mp4/tutorial_mscolab.mp4
+  .. video:: ../videos/mp4/tutorial_mscolab.mp4
      :alt: MSColab stores data in an online server, and can be used to access the data remotely as also working in a
            team where everyone contributes his part. It is used for collaborating with the users as a team together
            and working on a shared MSColab operation.

diff --git a/docs/tutorials/tutorial_msui_views.rst b/docs/tutorials/tutorial_msui_views.rst
@@ -4,7 +4,7 @@ Introduction to MSUI
 Using the different views of the MSUI with a fictitious flight path and demo data.
 
 
-  .. video:: ../_static/mp4//tutorial_views.mp4
+  .. video:: ../videos/mp4/tutorial_views.mp4
      :alt: Lets look at the tutorial of the various views required for flight planning:
            Top View (CTRL+H), Side View (CTRL+V), Linear View (CTRL+L) and Table View (CTRL+T).
            At first, lets open Top View and Side View.

diff --git a/docs/tutorials/tutorial_performance_settings.rst b/docs/tutorials/tutorial_performance_settings.rst
@@ -3,7 +3,7 @@ Table View and Aircraft Performance Data
 
 The range-specific data of an aircraft can be taken into account in Tableview for flight planning.
 
-  .. video:: ../_static/mp4/tutorial_performancesettings.mp4
+  .. video:: ../videos/mp4/tutorial_performancesettings.mp4
      :alt: This is the Performance Settings dockwidget opened in the Table View (CTRL+T) of the MSS software
            where by putting and changing some parameters, we can evaluate the performance of the aircraft.
            Parameters like Flight Altitude, Aviation fuel, Aircraft weight, Maximum take off weight

diff --git a/docs/tutorials/tutorial_remotesensing.rst b/docs/tutorials/tutorial_remotesensing.rst
@@ -4,7 +4,7 @@ Top View and Remotesensing Tools
 In order to be able to take into account the viewing angle and solar level for measuring instruments, the remotesensing tools are used
 
 
-  .. video:: ../_static/mp4/tutorial_remotesensing.mp4
+  .. video:: ../videos/mp4/tutorial_remotesensing.mp4
      :alt: This is the Remote Sensing Section of the Top View.
            It shows the position and angle of the flight from any particular object in the sky.
            Azimuth is the forward direction line of the flight. If we go above tHE AZIMUTH, angle is in positive,

diff --git a/docs/tutorials/tutorial_satellitetrack.rst b/docs/tutorials/tutorial_satellitetrack.rst
@@ -4,7 +4,7 @@ Top View and  Satellite Overflight
 To combine a flight path with a satellite overflight, the remotesensing widget is used.
 
 
-  .. video:: ../_static/mp4/tutorial_satellitetrack.mp4
+  .. video:: ../videos/mp4/tutorial_satellitetrack.mp4
      :alt: This is Satellite Tracking Prediction System that can be used to check the accuracy of the path
            travelled by a Satellite by the help of data collected from different space agencies and planning
            a flight accordingly.

diff --git a/docs/tutorials/tutorial_waypoints.rst b/docs/tutorials/tutorial_waypoints.rst
@@ -3,7 +3,7 @@ Top View Drawing Waypoints
 
 Waypoints for a flight path are defined, shifted and deleted.
 
-  .. video:: ../_static/mp4/tutorial_waypoints.mp4
+  .. video:: ../videos/mp4/tutorial_waypoints.mp4
      :alt: This is the waypoints tutorial,i.e., whenever we are going to plan
            a flight track, we have to place the waypoints in some places and,
            form a flight path.

diff --git a/localbuild/meta.yaml b/localbuild/meta.yaml
@@ -77,7 +77,7 @@ requirements:
     - metpy
     - pycountry
     - websocket-client
-    - libtiff <4.5.0
+    - libtiff
     - flask-wtf
     - email_validator
     - keyring

diff --git a/mslib/mscolab/app/__init__.py b/mslib/mscolab/app/__init__.py
@@ -25,6 +25,7 @@
 """
 
 import os
+import logging
 import sqlalchemy
 
 from flask_migrate import Migrate
@@ -34,7 +35,11 @@
 from flask import Flask, url_for
 from mslib.mscolab.conf import mscolab_settings
 from flask_sqlalchemy import SQLAlchemy
-from mslib.utils import prefix_route
+from mslib.utils import prefix_route, release_info
+
+message, update = release_info.check_for_new_release()
+if update:
+    logging.warning(message)
 
 
 DOCS_SERVER_PATH = os.path.dirname(os.path.abspath(mslib.__file__))
@@ -47,7 +52,7 @@
 APP.config.from_object(__name__)
 APP.route = prefix_route(APP.route, SCRIPT_NAME)
 
-APP.config['MSCOLAB_DATA_DIR'] = mscolab_settings.MSCOLAB_DATA_DIR
+APP.config['OPERATIONS_DATA'] = mscolab_settings.OPERATIONS_DATA
 APP.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI
 APP.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 APP.config['SQLALCHEMY_ECHO'] = mscolab_settings.SQLALCHEMY_ECHO