Add mypy Pre-commit Hook and Fix tools folder Pt.1 (#659)

* mypy for tools folder

* mypy check for tools folder

* xtce generation mypy update

* reworking the usage

* some mypy changes

* exclude type-arg check for list and dict

* mypy update for tools folder, completed

* small change

* updates for Et.Element

* merge conflict changes

.pre-commit-config.yaml

@@ -37,4 +37,13 @@ repos:
     hooks:
       - id: numpydoc-validation
         exclude: '^imap_processing/tests/|.*test.*'
-
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.10.0'
+    hooks:
+      - id: mypy
+        pass_filenames: false
+        args: [., --strict, --explicit-package-bases,
+                --disable-error-code, import-untyped,
+                --disable-error-code, import-not-found,
+                --disable-error-code, no-untyped-call,
+                --disable-error-code, type-arg]

pyproject.toml

@@ -116,3 +116,7 @@ checks = ["all", #report on all checks, except the following
     "ES01", # Ignore No extended summary found
     "RT02" ] # Ignore The first line of the Returns section
 exclude = ['__init__' ] # don't report on objects that match any of these regex
+
+[tool.mypy]
+follow_imports = 'skip'
+exclude = ["tests", "docs", "imap_processing"]

tools/metadata_generation/metadata_generator.py

@@ -1,12 +1,13 @@
 """SWAPI metadata generator."""
 
 from pathlib import Path
+from typing import Optional, Union
 
 import pandas as pd
 import yaml
 
 
-def get_global_attributes(sheet):
+def get_global_attributes(sheet: pd.DataFrame) -> pd.DataFrame:
     """
     Get the global attributes from a metadata spreadsheet.
 
@@ -17,7 +18,7 @@ def get_global_attributes(sheet):
 
     Returns
     -------
-    dict
+    pd.DataFrame
         A dictionary of the global attributes, where the keys are the attribute names
         and the values are the attribute descriptions.
     """
@@ -27,15 +28,17 @@ def get_global_attributes(sheet):
     return sheet.to_dict()["CATDESC"]
 
 
-def get_dataset_attributes(sheet, global_attrs=None):
+def get_dataset_attributes(
+    sheet: pd.DataFrame, global_attrs: Optional[dict] = None
+) -> dict:
     """
     Get the dataset attributes from a metadata spreadsheet.
 
     Parameters
     ----------
     sheet : pandas.DataFrame
         A sheet/tab from an Excel file represented as a pandas DataFrame.
-    global_attrs : list todo check
+    global_attrs : dict or None
         List of global attributes.
 
     Returns
@@ -64,7 +67,7 @@ def get_dataset_attributes(sheet, global_attrs=None):
 
 
 # Load all sheets
-def process_file(excel_path, output_folder):
+def process_file(excel_path: Union[str, Path], output_folder: Path) -> None:
     """
     Will process the metadata file and output the metadata to a JSON file.
 

tools/spice/spice_examples.py

@@ -8,13 +8,14 @@
 
 import logging
 
+import numpy as np
 import spiceypy as spice
 
 # Logger setup
 logger = logging.getLogger(__name__)
 
 
-def get_attitude_timerange(ck_kernel, id):
+def get_attitude_timerange(ck_kernel: str, id: int) -> tuple:
     """
     Get attitude timerange using the ck kernel.
 
@@ -65,7 +66,9 @@ def get_attitude_timerange(ck_kernel, id):
     return start, end
 
 
-def _get_particle_velocity(direct_events):
+def _get_particle_velocity(
+    direct_events: dict,
+) -> np.ndarray:
     """
     Get the particle velocity in the heliosphere frame.
 
@@ -76,7 +79,7 @@ def _get_particle_velocity(direct_events):
 
     Returns
     -------
-    vultra_heliosphere_frame : np.ndarray
+    vultra_heliosphere_frame : numpy.ndarray
         Particle velocity in the heliosphere frame.
 
     Notes
@@ -108,7 +111,7 @@ def _get_particle_velocity(direct_events):
     return ultra_velocity_heliosphere_frame
 
 
-def build_annotated_events(direct_events, kernels):
+def build_annotated_events(direct_events: dict, kernels: list) -> None:
     """
     Build annotated events.
 

tools/spice/spice_utils.py

@@ -2,13 +2,16 @@
 
 import logging
 import os
+from typing import Optional
 
 import spiceypy as spice
 
 logger = logging.getLogger(__name__)
 
 
-def list_files_with_extensions(directory: str, extensions=None) -> list[str]:
+def list_files_with_extensions(
+    directory: str, extensions: Optional[list[str]] = None
+) -> list[str]:
     """
     List all files in a given directory that have the specified extensions.
 
@@ -41,7 +44,7 @@ def list_files_with_extensions(directory: str, extensions=None) -> list[str]:
     return matching_files
 
 
-def list_loaded_kernels(extensions=None) -> list:
+def list_loaded_kernels(extensions: Optional[list[str]] = None) -> list:
     """
     List furnished spice kernels, optionally filtered by specific extensions.
 

tools/xtce_generation/ccsds_header_xtce_generator.py

@@ -9,7 +9,7 @@
 class CCSDSParameters:
     """The defined class for CCSDS parameters. Contains the object itself."""
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.parameters = [
             {
                 "name": "VERSION",

tools/xtce_generation/telemetry_generator.py

@@ -2,6 +2,7 @@
 
 import xml.etree.ElementTree as Et
 from datetime import datetime
+from typing import Optional
 
 import pandas as pd
 
@@ -39,7 +40,13 @@ class TelemetryGenerator:
         Default set to None.
     """
 
-    def __init__(self, packet_name, path_to_excel_file, apid, pkt=None):
+    def __init__(
+        self,
+        packet_name: str,
+        path_to_excel_file: str,
+        apid: int,
+        pkt: Optional[str] = None,
+    ) -> None:
         """Initialize TelemetryGenerator."""
         self.packet_name = packet_name
         self.apid = apid
@@ -52,7 +59,7 @@ def __init__(self, packet_name, path_to_excel_file, apid, pkt=None):
         else:
             self.pkt = pkt
 
-    def create_telemetry_xml(self):
+    def create_telemetry_xml(self) -> tuple:
         """
         Create an XML representation of telemetry data based on input parameters.
 
@@ -97,7 +104,7 @@ def create_telemetry_xml(self):
 
         return root, parameter_type_set, parameter_set, telemetry_metadata
 
-    def get_unique_bits_length(self):
+    def get_unique_bits_length(self) -> dict:
         """
         Create dictionary.
 
@@ -135,7 +142,11 @@ def get_unique_bits_length(self):
         unique_lengths = dict(sorted(unique_lengths.items(), key=lambda item: item[1]))
         return unique_lengths
 
-    def create_parameter_types(self, parameter_type_set, unique_lengths):
+    def create_parameter_types(
+        self,
+        parameter_type_set: Et.Element,
+        unique_lengths: dict,
+    ) -> Et.Element:
         """
         Create parameter types based on 'dataType' for the unique 'lengthInBits' values.
 
@@ -144,20 +155,21 @@ def create_parameter_types(self, parameter_type_set, unique_lengths):
 
         Parameters
         ----------
-        parameter_type_set : list TODO Double Check
+        parameter_type_set : Et.Element
             The ParameterTypeSet element where parameter types are.
-        unique_lengths : list TODO Double Check
+        unique_lengths : dict
             Unique values from the 'lengthInBits' column.
 
         Returns
         -------
-        parameter_type_set
+        parameter_type_set : Et.Element
             The updated ParameterTypeSet element.
         """
         for parameter_type_ref_name, size in unique_lengths.items():
             if "UINT" in parameter_type_ref_name:
                 parameter_type = Et.SubElement(
-                    parameter_type_set, "xtce:IntegerParameterType"
+                    parameter_type_set,
+                    "xtce:IntegerParameterType",
                 )
                 parameter_type.attrib["name"] = parameter_type_ref_name
                 parameter_type.attrib["signed"] = "false"
@@ -168,7 +180,8 @@ def create_parameter_types(self, parameter_type_set, unique_lengths):
 
             elif any(x in parameter_type_ref_name for x in ["SINT", "INT"]):
                 parameter_type = Et.SubElement(
-                    parameter_type_set, "xtce:IntegerParameterType"
+                    parameter_type_set,
+                    "xtce:IntegerParameterType",
                 )
                 parameter_type.attrib["name"] = parameter_type_ref_name
                 parameter_type.attrib["signed"] = "true"
@@ -178,7 +191,8 @@ def create_parameter_types(self, parameter_type_set, unique_lengths):
 
             elif "BYTE" in parameter_type_ref_name:
                 binary_parameter_type = Et.SubElement(
-                    parameter_type_set, "xtce:BinaryParameterType"
+                    parameter_type_set,
+                    "xtce:BinaryParameterType",
                 )
                 binary_parameter_type.attrib["name"] = parameter_type_ref_name
 
@@ -195,20 +209,22 @@ def create_parameter_types(self, parameter_type_set, unique_lengths):
 
         return parameter_type_set
 
-    def create_ccsds_packet_parameters(self, parameter_set, ccsds_parameters):
+    def create_ccsds_packet_parameters(
+        self, parameter_set: Et.Element, ccsds_parameters: list
+    ) -> Et.Element:
         """
         Create XML elements to define CCSDS packet parameters based on the given data.
 
         Parameters
         ----------
-        parameter_set : list
+        parameter_set : Et.Element
             The ParameterSet element where parameters will be added.
-        ccsds_parameters : dict
+        ccsds_parameters : list
             A list of dictionaries containing CCSDS parameter data.
 
         Returns
         -------
-        parameter_set
+        parameter_set : Et.Element
             The updated ParameterSet element.
         """
         for parameter_data in ccsds_parameters:
@@ -222,8 +238,11 @@ def create_ccsds_packet_parameters(self, parameter_set, ccsds_parameters):
         return parameter_set
 
     def create_container_set(
-        self, telemetry_metadata, ccsds_parameters, container_name
-    ):
+        self,
+        telemetry_metadata: Et.Element,
+        ccsds_parameters: list,
+        container_name: str,
+    ) -> Et.Element:
         """
         Create XML elements.
 
@@ -232,7 +251,7 @@ def create_container_set(
 
         Parameters
         ----------
-        telemetry_metadata : dict todo check
+        telemetry_metadata : Et.Element
             The TelemetryMetaData element where containers are.
         ccsds_parameters : list
             A list of dictionaries containing CCSDS parameter data.
@@ -241,12 +260,11 @@ def create_container_set(
 
         Returns
         -------
-        telemetry_metadata
+        telemetry_metadata : Et.Element
             The updated TelemetryMetaData element.
         """
         # Create ContainerSet element
         container_set = Et.SubElement(telemetry_metadata, "xtce:ContainerSet")
-
         # Create CCSDSPacket SequenceContainer
         ccsds_packet_container = Et.SubElement(container_set, "xtce:SequenceContainer")
         ccsds_packet_container.attrib["name"] = "CCSDSPacket"
@@ -288,20 +306,20 @@ def create_container_set(
 
         return telemetry_metadata
 
-    def create_remaining_parameters(self, parameter_set):
+    def create_remaining_parameters(self, parameter_set: Et.Element) -> Et.Element:
         """
         Create XML elements for parameters.
 
         These are based on DataFrame rows starting from SHCOARSE (also known as MET).
 
         Parameters
         ----------
-        parameter_set : list todo check
+        parameter_set : Et.Element
             The ParameterSet element where parameters will be added.
 
         Returns
         -------
-        parameter_set
+        parameter_set : Et.Element
             The updated ParameterSet element.
         """
         # Process rows from SHCOARSE until the last available row in the DataFrame
@@ -326,7 +344,7 @@ def create_remaining_parameters(self, parameter_set):
 
         return parameter_set
 
-    def generate_telemetry_xml(self, output_xml_path, container_name):
+    def generate_telemetry_xml(self, output_xml_path: str, container_name: str) -> None:
         """
         Create and output an XTCE file based on the data within the class.
 

tools/xtce_generation/xtce_generator_codice.py

@@ -11,7 +11,7 @@
 from tools.xtce_generation.telemetry_generator import TelemetryGenerator
 
 
-def main():
+def main() -> None:
     """Function used by instrument to generate XTCE."""
     instrument_name = "codice"
     current_directory = Path(__file__).parent

tools/xtce_generation/xtce_generator_glows.py

@@ -11,7 +11,7 @@
 from tools.xtce_generation.telemetry_generator import TelemetryGenerator
 
 
-def main():
+def main() -> None:
     """
     Function is used to generate the GLOWS XTCE files for packet processing.
 

tools/xtce_generation/xtce_generator_hi.py

@@ -11,7 +11,7 @@
 from tools.xtce_generation.telemetry_generator import TelemetryGenerator
 
 
-def main():
+def main() -> None:
     """Function used by hi to generate XTCE."""
 
     instrument_name = "hi"

tools/xtce_generation/xtce_generator_hit.py

@@ -13,7 +13,7 @@
 from imap_processing import imap_module_directory
 
 
-def main():
+def main() -> None:
     """Function used by hit to generate XTCE."""
     instrument_name = "hit"
     current_directory = Path(__file__).parent

tools/xtce_generation/xtce_generator_lo.py

@@ -18,7 +18,7 @@
 # P_ILO_SCI_DE
 
 
-def main():
+def main() -> None:
     """Function used by lo to generate XTCE."""
     instrument_name = "lo"
     current_directory = Path(__file__).parent