From a38649c7a5ffef5b440b76318583af4c165bc4ce Mon Sep 17 00:00:00 2001 From: Tenzin Choedon <36522642+tech3371@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:47:26 -0600 Subject: [PATCH] Xtce generator updates (#155) - updated xtce generator code to handle different data types. made code more generic. - fixed XTCE header schema - added template that all instrument can use - moved past xml file into their respective folder and fixed path with new path for IDEX and SWE --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- imap_processing/__init__.py | 3 - imap_processing/codice/L0/hi_pha.py | 32 --- imap_processing/codice/L0/lo_pha.py | 32 --- .../codice/L0/nsw_species_counts.py | 32 --- .../codice/L0/omni_species_counts.py | 32 --- .../codice/L0/sw_species_counts.py | 32 --- .../P_COD_HI_OMNI_SPECIES_COUNTS.xml | 179 +++++++++++++ .../packet_definitions/P_COD_HI_PHA.xml | 159 +++++++++++ .../P_COD_LO_NSW_SPECIES_COUNTS.xml | 179 +++++++++++++ .../packet_definitions/P_COD_LO_PHA.xml | 161 +++++++++++ .../P_COD_LO_SW_SPECIES_COUNTS.xml | 179 +++++++++++++ imap_processing/idex/idex_packet_parser.py | 6 +- .../idex_packet_definition.xml | 0 .../packet_definitions/codice/hi_pha.xml | 141 ---------- .../codice/lo_nsw_species_counts.xml | 163 ------------ .../packet_definitions/codice/lo_pha.xml | 141 ---------- .../codice/lo_sw_species_counts.xml | 163 ------------ .../codice/omni_species_counts.xml | 155 ----------- imap_processing/swe/decom_swe.py | 6 +- .../swe/packet_definitions/__init__.py | 0 .../swe_packet_definition.xml | 0 poetry.lock | 82 +++++- .../ccsds_header_xtce_generator.py | 16 +- tools/xtce_generation/telemetry_generator.py | 249 ++++++++---------- .../xtce_generator_template.py | 50 ++++ 25 files changed, 1107 insertions(+), 1085 deletions(-) delete mode 100644 imap_processing/codice/L0/hi_pha.py delete mode 100644 imap_processing/codice/L0/lo_pha.py delete mode 100644 imap_processing/codice/L0/nsw_species_counts.py delete mode 100644 imap_processing/codice/L0/omni_species_counts.py delete mode 100644 imap_processing/codice/L0/sw_species_counts.py create mode 100644 imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml create mode 100644 imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml create mode 100644 imap_processing/codice/packet_definitions/P_COD_LO_NSW_SPECIES_COUNTS.xml create mode 100644 imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml create mode 100644 imap_processing/codice/packet_definitions/P_COD_LO_SW_SPECIES_COUNTS.xml rename imap_processing/{ => idex}/packet_definitions/idex_packet_definition.xml (100%) delete mode 100644 imap_processing/packet_definitions/codice/hi_pha.xml delete mode 100644 imap_processing/packet_definitions/codice/lo_nsw_species_counts.xml delete mode 100644 imap_processing/packet_definitions/codice/lo_pha.xml delete mode 100644 imap_processing/packet_definitions/codice/lo_sw_species_counts.xml delete mode 100644 imap_processing/packet_definitions/codice/omni_species_counts.xml create mode 100644 imap_processing/swe/packet_definitions/__init__.py rename imap_processing/{ => swe}/packet_definitions/swe_packet_definition.xml (100%) create mode 100644 tools/xtce_generation/xtce_generator_template.py diff --git a/imap_processing/__init__.py b/imap_processing/__init__.py index 94ea98581..0fc00b797 100644 --- a/imap_processing/__init__.py +++ b/imap_processing/__init__.py @@ -8,6 +8,3 @@ # Eg. imap_module_directory = /usr/local/lib/python3.11/site-packages/imap_processing imap_module_directory = Path(__file__).parent - -# Relative to imap_module_directory, set path of packet definitions directory. -packet_definition_directory = f"{imap_module_directory}/packet_definitions/" diff --git a/imap_processing/codice/L0/hi_pha.py b/imap_processing/codice/L0/hi_pha.py deleted file mode 100644 index 00e73cd0e..000000000 --- a/imap_processing/codice/L0/hi_pha.py +++ /dev/null @@ -1,32 +0,0 @@ -"""This is the xtce generator for the P_COD_HI_PHA packet.""" - -import pandas as pd - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main(): - packet_name = "P_COD_HI_PHA" - path_to_excel_file = ( - "/Users/gamo6782/Desktop/IMAP/TLM_COD_20230629-110638(update).xlsx" - ) - apid = "1169" - sci_byte = 276480 - - xls = pd.ExcelFile(path_to_excel_file) - pkt = xls.parse(packet_name) - - # Specify the desired output XML path - output_xml_path = "../../packet_definitions/codice/hi_pha.xml" - - # Create an instance of the TelemetryGenerator class - generator = TelemetryGenerator(packet_name, path_to_excel_file, apid, sci_byte, pkt) - - # Generate the telemetry XML with the provided output path - generator.generate_telemetry_xml( - output_xml_path, ["Data", "EventData"], "CoDICESciencePacket" - ) - - -if __name__ == "__main__": - main() diff --git a/imap_processing/codice/L0/lo_pha.py b/imap_processing/codice/L0/lo_pha.py deleted file mode 100644 index cfd79d3ea..000000000 --- a/imap_processing/codice/L0/lo_pha.py +++ /dev/null @@ -1,32 +0,0 @@ -"""This is the xtce generator for the P_COD_LO_PHA packet.""" - -import pandas as pd - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main(): - packet_name = "P_COD_LO_PHA" - path_to_excel_file = ( - "/Users/gamo6782/Desktop/IMAP/TLM_COD_20230629-110638(update).xlsx" - ) - apid = "1153" - sci_byte = 276480 - - xls = pd.ExcelFile(path_to_excel_file) - pkt = xls.parse(packet_name) - - # Specify the desired output XML path - output_xml_path = "../../packet_definitions/codice/lo_pha.xml" - - # Create an instance of the TelemetryGenerator class - generator = TelemetryGenerator(packet_name, path_to_excel_file, apid, sci_byte, pkt) - - # Generate the telemetry XML with the provided output path - generator.generate_telemetry_xml( - output_xml_path, ["Data", "EventData"], "CoDICESciencePacket" - ) - - -if __name__ == "__main__": - main() diff --git a/imap_processing/codice/L0/nsw_species_counts.py b/imap_processing/codice/L0/nsw_species_counts.py deleted file mode 100644 index 1e530a435..000000000 --- a/imap_processing/codice/L0/nsw_species_counts.py +++ /dev/null @@ -1,32 +0,0 @@ -"""This is the xtce generator for the P_COD_LO_NSW_SPECIES_COUNTS packet.""" - -import pandas as pd - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main(): - packet_name = "P_COD_LO_NSW_SPECIES_COUNTS" - path_to_excel_file = ( - "/Users/gamo6782/Desktop/IMAP/TLM_COD_20230629-110638(update).xlsx" - ) - apid = "1157" - sci_byte = 37748736 - - xls = pd.ExcelFile(path_to_excel_file) - pkt = xls.parse(packet_name) - - # Specify the desired output XML path - output_xml_path = "../../packet_definitions/codice/lo_nsw_species_counts.xml" - - # Create an instance of the TelemetryGenerator class - generator = TelemetryGenerator(packet_name, path_to_excel_file, apid, sci_byte, pkt) - - # Generate the telemetry XML with the provided output path - generator.generate_telemetry_xml( - output_xml_path, ["Data", "EventData"], "CoDICESciencePacket" - ) - - -if __name__ == "__main__": - main() diff --git a/imap_processing/codice/L0/omni_species_counts.py b/imap_processing/codice/L0/omni_species_counts.py deleted file mode 100644 index 0b412caca..000000000 --- a/imap_processing/codice/L0/omni_species_counts.py +++ /dev/null @@ -1,32 +0,0 @@ -"""This is the xtce generator for the P_COD_HI_OMNI_SPECIES_COUNTS packet.""" - -import pandas as pd - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main(): - packet_name = "P_COD_HI_OMNI_SPECIES_COUNTS" - path_to_excel_file = ( - "/Users/gamo6782/Desktop/IMAP/TLM_COD_20230629-110638(update).xlsx" - ) - apid = "1172" - sci_byte = 328704768 - - xls = pd.ExcelFile(path_to_excel_file) - pkt = xls.parse(packet_name) - - # Specify the desired output XML path - output_xml_path = "../../packet_definitions/codice/omni_species_counts.xml" - - # Create an instance of the TelemetryGenerator class - generator = TelemetryGenerator(packet_name, path_to_excel_file, apid, sci_byte, pkt) - - # Generate the telemetry XML with the provided output path - generator.generate_telemetry_xml( - output_xml_path, ["Data", "EventData"], "CoDICESciencePacket" - ) - - -if __name__ == "__main__": - main() diff --git a/imap_processing/codice/L0/sw_species_counts.py b/imap_processing/codice/L0/sw_species_counts.py deleted file mode 100644 index cdd1ef4d0..000000000 --- a/imap_processing/codice/L0/sw_species_counts.py +++ /dev/null @@ -1,32 +0,0 @@ -"""This is the xtce generator for the P_COD_LO_SW_SPECIES_COUNTS packet.""" - -import pandas as pd - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main(): - packet_name = "P_COD_LO_SW_SPECIES_COUNTS" - path_to_excel_file = ( - "/Users/gamo6782/Desktop/IMAP/TLM_COD_20230629-110638(update).xlsx" - ) - apid = "1156" - sci_byte = 37748736 - - xls = pd.ExcelFile(path_to_excel_file) - pkt = xls.parse(packet_name) - - # Specify the desired output XML path - output_xml_path = "../../packet_definitions/codice/lo_sw_species_counts.xml" - - # Create an instance of the TelemetryGenerator class - generator = TelemetryGenerator(packet_name, path_to_excel_file, apid, sci_byte, pkt) - - # Generate the telemetry XML with the provided output path - generator.generate_telemetry_xml( - output_xml_path, ["Data", "EventData"], "CoDICESciencePacket" - ) - - -if __name__ == "__main__": - main() diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml new file mode 100644 index 000000000..0e975b5c8 --- /dev/null +++ b/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 28704768 + + + + + + + CCSDS Packet Version Number (always 0) + + + CCSDS Packet Type Indicator (0=telemetry) + + + CCSDS Packet Secondary Header Flag (always 1) + + + CCSDS Packet Application Process ID + + + CCSDS Packet Grouping Flags (3=not part of group) + + + CCSDS Packet Sequence Count (increments with each new packet) + + + CCSDS Packet Length (number of bytes after Packet length minus 1) + + + CCSDS Packet Time Stamp (coarse time) + + + Packet version - this will be incremented each time the format of the packet changes. + + + Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + + Full-seconds portion of the time at which the 16-spin cycle started + + + Sub-seconds portion of the time at which the 16-spin cycle started (microseconds) + + + Spare for alignment + + + Unique ID assigned to a specific table configuration. This field is used to link the overall acquisition and processing settings to a specific table configuration. + + + Plan table that was in use + + + Plan step that was active when this data was acquired and processed. + + + View ID provides information about how data was collapsed and/or compressed. + + + N/A for CoDICE-Hi; included in order to maintain common packet format + + + N/A for CoDICE-Hi; included in order to maintain common packet format + + + N/A for CoDICE-Hi; included in order to maintain common packet format + + + Indicates that there was some error detected during acquisition or processing of the data. Errors could include corrupted acquisition memory (i.e. EDAC errors), timing violations, or other events that interrupted or otherwise affected data collection. + + + Whether/how the data is compressed. + + + Number of bytes in the Data array. If compressed, this value represents the length of the compressed data. + + + Counter Data + +Variable Length; Maximum (based on uncollapsed, uncompressed data, and assuming all 146 sqrt(2) speceis counters included): + +16 spins x 16 positions x 24 spin-angles x 32 bits x 146 counters = 28,704,768 bits (3,588,096 bytes) + +Nominal (based on current plan for counter selection, collapsing, and compression): 42,240 bits (5,281 bytes) + +Data format is a series of spin-angle x position x spin number data cubes collapsed per the SCI_LUT Collapse Table selected by the View_ID. Which counters are included is determined by using the Plan_ID and Plan_Step to index into the SCI_LUT Data Products Hi/Lo tables to find all the counters that are associated with the View_ID. + +The collapsed data cubes are also optionally compressed using Lossy and/or Lossless Compression. Lossy compression is a table-based 24->8 bit compression applied to each counter value. Lossless compression uses the LZMA compression algorithm and is applied to the full Data field as a single unit. + +Field will additionally be padded in order to meet the requirement of packets being a multiple of 16 bits; any pad bits will be accounted for in the CCSDS header Length field, but will *not* be included in the Byte_Count field + +When this array is too large for a single CCSDS packet, CoDICE will utilize the CCSDS Grouping flags to provide the full data packet over several CCSDS packets. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml b/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml new file mode 100644 index 000000000..01be9a44f --- /dev/null +++ b/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml @@ -0,0 +1,159 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 276480 + + + + + + + CCSDS Packet Version Number (always 0) + + + CCSDS Packet Type Indicator (0=telemetry) + + + CCSDS Packet Secondary Header Flag (always 1) + + + CCSDS Packet Application Process ID + + + CCSDS Packet Grouping Flags (3=not part of group) + + + CCSDS Packet Sequence Count (increments with each new packet) + + + CCSDS Packet Length (number of bytes after Packet length minus 1) + + + CCSDS Packet Time Stamp (coarse time) + + + Packet version - this will be incremented each time the format of the packet changesCOD_LO_PHA. + + + Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + + Full-seconds portion of the time at which the 16-spin cycle started + + + Sub-seconds portion of the time at which the 16-spin cycle started + + + Spare for alignment + + + Total number of TCR events that occurred during the collection cycle + + + Total number of DCR events that occurred during the collection cycle + + + Total number of APD-Only events that occurred during the collection cycle + + + Number of events selected for downlink (i.e. number of events in the Event_Data array) + + + Whether the event data is compressed. If 1/Yes, Event_Data array is compressed using the Rice compression algorithm. + + + Number of bytes in the Event_Data array. If compressed, this value represents the length of the compressed data. + + + Optionally compressed array of Event Data + +Format is TBD; some considerations/options: +- Full events have a lot of redundant data (e.g. will have many events with the same priority/spin/spin phase information). How well does compression to deal with the redundancy? +- Could include mini-headers for each (priority,spin, spin-phase) group and strip the redundant data from the events +- Should events be tightly packed, or can we pad out to 64-bit word boundaries? How well does compression compensate for the extra bits? + +Each event consists of: +- 10-bit TOF +- 9-bit SSD Energy +- 2-bit Energy Range +- 7-bit Spin Angle +- 4-bit SSD Position +- 4-bit Spin Number +- 2-bit PHA Type + +TBD: Events may be tightly packed, or may have spares added to keep each event byte-aligned. In either case, there may be up to 1 byte of padding to keep the total size of the packet even. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/imap_processing/codice/packet_definitions/P_COD_LO_NSW_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_LO_NSW_SPECIES_COUNTS.xml new file mode 100644 index 000000000..41c3ed886 --- /dev/null +++ b/imap_processing/codice/packet_definitions/P_COD_LO_NSW_SPECIES_COUNTS.xml @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 37748736 + + + + + + + CCSDS Packet Version Number (always 0) + + + CCSDS Packet Type Indicator (0=telemetry) + + + CCSDS Packet Secondary Header Flag (always 1) + + + CCSDS Packet Application Process ID + + + CCSDS Packet Grouping Flags (3=not part of group) + + + CCSDS Packet Sequence Count (increments with each new packet) + + + CCSDS Packet Length (number of bytes after Packet length minus 1) + + + CCSDS Packet Time Stamp (coarse time) + + + Packet version - this will be incremented each time the format of the packet changes. + + + Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + + Full-seconds portion of the time at which the 16-spin cycle started + + + Sub-seconds portion of the time at which the 16-spin cycle started (microseconds) + + + Spare for alignment + + + Unique ID assigned to a specific table configuration. This field is used to link the overall acquisition and processing settings to a specific table configuration. + + + Plan table that was in use + + + Plan step that was active when this data was acquired and processed. + + + View ID provides information about how data was collapsed and/or compressed. + + + Indicates the point when Reduced Gain Factor Operation (RGFO) was actived. In RGFO, the Entrance ESA voltage is reduced in order to limit the number of ions that reach the detectors. + + + Indicates the point when No-Scan Operation (NSO) was actived. In NSO, the ESA voltage is set to the first step in the scan and remains fixed until the next cycle boundary. + + + Indicates whether FSW is tracking the High-Gain bias curve or the Low-Gain bias curve. + + + Indicates that there was some error detected during acquisition or processing of the data. Errors could include corrupted acquisition memory (i.e. EDAC errors), timing violations, or other events that interrupted or otherwise affected data collection. + + + Whether/how the data is compressed. + + + Number of bytes in the Data array. If compressed, this value represents the length of the compressed data. + + + Counter Data + +Variable Length; Maximum (based on uncollapsed, uncompressed data, and assuming all 32 species-counters included): + +128 energies x 24 positions x 12 spin-angles x 32 bits x 32 counters = 37,748,736 bits (4,718,592 bytes) + +Realistically, data is aggressively collapsed and compressed, and only a subset of the 32 species counters will be included, so this data field will be much smaller than the maximum. + +Data format is a series of spin-angle x position x energy data cubes collapsed per the SCI_LUT Collapse Table selected by the View_ID. Which counters are included is determined by using the Plan_ID and Plan_Step to index into the SCI_LUT Data Products Hi/Lo tables to find all the counters that are associated with the View_ID. + +The collapsed data cubes are also optionally compressed using Lossy and/or Lossless Compression. Lossy compression is a table-based 24->8 bit compression applied to each counter value. Lossless compression uses the LZMA compression algorithm and is applied to the full Data field as a single unit. + +Field will additionally be padded in order to meet the requirement of packets being a multiple of 16 bits; any pad bits will be accounted for in the CCSDS header Length field, but will *not* be included in the Byte_Count field + +When this array is too large for a single CCSDS packet, CoDICE will utilize the CCSDS Grouping flags to provide the full data packet over several CCSDS packets. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml b/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml new file mode 100644 index 000000000..4ab88f175 --- /dev/null +++ b/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 276480 + + + + + + + CCSDS Packet Version Number (always 0) + + + CCSDS Packet Type Indicator (0=telemetry) + + + CCSDS Packet Secondary Header Flag (always 1) + + + CCSDS Packet Application Process ID + + + CCSDS Packet Grouping Flags (3=not part of group) + + + CCSDS Packet Sequence Count (increments with each new packet) + + + CCSDS Packet Length (number of bytes after Packet length minus 1) + + + CCSDS Packet Time Stamp (coarse time) + + + Packet version - this will be incremented each time the format of the packet changesCOD_LO_PHA. + + + Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + + Full-seconds portion of the time at which the 16-spin cycle started + + + Sub-seconds portion of the time at which the 16-spin cycle started + + + Spare for alignment + + + Total number of TCR events that occurred during the collection cycle + + + Total number of DCR events that occurred during the collection cycle + + + Total number of APD-Only events that occurred during the collection cycle + + + Number of events selected for downlink (i.e. number of events in the Event_Data array) + + + Whether the event data is compressed. If 1/Yes, Event_Data array is compressed using the LZMA compression algorithm. + + + Number of bytes in the Event_Data array. If compressed, this value represents the length of the compressed data. + + + Optionally compressed array of Event Data + +Format is TBD; some considerations/options: +- Full events have a lot of redundant data (e.g. will have many events with the same priority/e-step/spin phase information). How well does compression to deal with the redundancy? +- Could include mini-headers for each (priority,e-step, spin-phase) group and strip the redundant data from the events +- Should events be tightly packed, or can we pad out to 64-bit word boundaries? How well does compression compensate for the extra bits? + +Each event consists of: +- 7-bit E-step +- 10-bit TOF +- 9-bit APD Energy +- 7-bit Spin Angle +- 5-bit Position +- 5-bit APD-ID +- 1-bit APD-Gain +- 2-bit PHA Type +- 3-bit Priority Range + +TBD: Events may be tightly packed, or may have spares added to keep each event byte-aligned. In either case, there may be up to 1 byte of padding to keep the total size of the packet even. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/imap_processing/codice/packet_definitions/P_COD_LO_SW_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_LO_SW_SPECIES_COUNTS.xml new file mode 100644 index 000000000..b880a16d6 --- /dev/null +++ b/imap_processing/codice/packet_definitions/P_COD_LO_SW_SPECIES_COUNTS.xml @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 37748736 + + + + + + + CCSDS Packet Version Number (always 0) + + + CCSDS Packet Type Indicator (0=telemetry) + + + CCSDS Packet Secondary Header Flag (always 1) + + + CCSDS Packet Application Process ID + + + CCSDS Packet Grouping Flags (3=not part of group) + + + CCSDS Packet Sequence Count (increments with each new packet) + + + CCSDS Packet Length (number of bytes after Packet length minus 1) + + + CCSDS Packet Time Stamp (coarse time) + + + Packet version - this will be incremented each time the format of the packet changes. + + + Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + + Full-seconds portion of the time at which the 16-spin cycle started + + + Sub-seconds portion of the time at which the 16-spin cycle started (microseconds) + + + Spare for alignment + + + Unique ID assigned to a specific table configuration. This field is used to link the overall acquisition and processing settings to a specific table configuration. + + + Plan table that was in use + + + Plan step that was active when this data was acquired and processed. + + + View ID provides information about how data was collapsed and/or compressed. + + + Indicates the point when Reduced Gain Factor Operation (RGFO) was actived. In RGFO, the Entrance ESA voltage is reduced in order to limit the number of ions that reach the detectors. + + + Indicates the point when No-Scan Operation (NSO) was actived. In NSO, the ESA voltage is set to the first step in the scan and remains fixed until the next cycle boundary. + + + Indicates whether FSW is tracking the High-Gain bias curve or the Low-Gain bias curve. + + + Indicates that there was some error detected during acquisition or processing of the data. Errors could include corrupted acquisition memory (i.e. EDAC errors), timing violations, or other events that interrupted or otherwise affected data collection. + + + Whether/how the data is compressed. + + + Number of bytes in the Data array. If compressed, this value represents the length of the compressed data. + + + Counter Data + +Variable Length; Maximum (based on uncollapsed, uncompressed data, and assuming all 32 species-counters included): + +128 energies x 24 positions x 12 spin-angles x 32 bits x 32 counters = 37,748,736 bits (4,718,592 bytes) + +Realistically, data is aggressively collapsed and compressed, and only a subset of the 32 species counters will be included, so this data field will be much smaller than the maximum. + +Data format is a series of spin-angle x position x energy data cubes collapsed per the SCI_LUT Collapse Table selected by the View_ID. Which counters are included is determined by using the Plan_ID and Plan_Step to index into the SCI_LUT Data Products Hi/Lo tables to find all the counters that are associated with the View_ID. + +The collapsed data cubes are also optionally compressed using Lossy and/or Lossless Compression. Lossy compression is a table-based 24->8 bit compression applied to each counter value. Lossless compression uses the LZMA compression algorithm and is applied to the full Data field as a single unit. + +Field will additionally be padded in order to meet the requirement of packets being a multiple of 16 bits; any pad bits will be accounted for in the CCSDS header Length field, but will *not* be included in the Byte_Count field + +When this array is too large for a single CCSDS packet, CoDICE will utilize the CCSDS Grouping flags to provide the full data packet over several CCSDS packets. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/imap_processing/idex/idex_packet_parser.py b/imap_processing/idex/idex_packet_parser.py index a2b54dc49..30d4b19a3 100644 --- a/imap_processing/idex/idex_packet_parser.py +++ b/imap_processing/idex/idex_packet_parser.py @@ -5,7 +5,7 @@ import xarray as xr from space_packet_parser import parser, xtcedef -from imap_processing import packet_definition_directory +from imap_processing import imap_module_directory TWENTY_MICROSECONDS = 20 * (10 ** (-6)) @@ -57,8 +57,8 @@ def __init__(self, packet_file: str): ----- Currently assumes one L0 file will generate exactly one l1a file """ - - xtce_file = f"{packet_definition_directory}/idex_packet_definition.xml" + xtce_filename = "idex_packet_definition.xml" + xtce_file = f"{imap_module_directory}/idex/packet_definitions/{xtce_filename}" packet_definition = xtcedef.XtcePacketDefinition(xtce_document=xtce_file) packet_parser = parser.PacketParser(packet_definition) diff --git a/imap_processing/packet_definitions/idex_packet_definition.xml b/imap_processing/idex/packet_definitions/idex_packet_definition.xml similarity index 100% rename from imap_processing/packet_definitions/idex_packet_definition.xml rename to imap_processing/idex/packet_definitions/idex_packet_definition.xml diff --git a/imap_processing/packet_definitions/codice/hi_pha.xml b/imap_processing/packet_definitions/codice/hi_pha.xml deleted file mode 100644 index 624a184f4..000000000 --- a/imap_processing/packet_definitions/codice/hi_pha.xml +++ /dev/null @@ -1,141 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 276480 - - - - - - - CCSDS Packet Version Number (always 0) - - - CCSDS Packet Type Indicator (0=telemetry) - - - CCSDS Packet Secondary Header Flag (always 1) - - - CCSDS Packet Application Process ID - - - CCSDS Packet Grouping Flags (3=not part of group) - - - CCSDS Packet Sequence Count (increments with each new packet) - - - CCSDS Packet Length (number of bytes after Packet length minus 1) - - - CCSDS Packet Time Stamp (coarse time) - - - Spin Period reported by the Spacecraft - - - Acquisition Start Time (Seconds) - - - Acquisition Start Time (Subseconds) - - - Spare for alignment - - - Total number of TCR events - - - Total number of DCR events - - - Total number of APD-Only events - - - Number of events included in this packet - - - Whether the event data is compressed - - - Number of bytes in the Event Data array - - - Event Data - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/imap_processing/packet_definitions/codice/lo_nsw_species_counts.xml b/imap_processing/packet_definitions/codice/lo_nsw_species_counts.xml deleted file mode 100644 index badad7a28..000000000 --- a/imap_processing/packet_definitions/codice/lo_nsw_species_counts.xml +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 37748736 - - - - - - - CCSDS Packet Version Number (always 0) - - - CCSDS Packet Type Indicator (0=telemetry) - - - CCSDS Packet Secondary Header Flag (always 1) - - - CCSDS Packet Application Process ID - - - CCSDS Packet Grouping Flags (3=not part of group) - - - CCSDS Packet Sequence Count (increments with each new packet) - - - CCSDS Packet Length (number of bytes after Packet length minus 1) - - - CCSDS Packet Time Stamp (coarse time) - - - Spin Period reported by the Spacecraft - - - Acquisition Start Time (Seconds) - - - Acquisition Start Time (Subseconds) - - - Spare for alignment - - - Science Lookup Table Version/ID - - - Plan Table ID - - - Plan Step Number - - - View table used for data collapsing and compression - - - Half-spin when Reduced Gain Factor Operation was activated - - - Half-spin when No Scan Operation was activated - - - Bias Voltage Mode - - - Indicates a data quality issue - - - Compression Configuration - - - Number of bytes in the Data array - - - Data Array - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/imap_processing/packet_definitions/codice/lo_pha.xml b/imap_processing/packet_definitions/codice/lo_pha.xml deleted file mode 100644 index b44e8dc39..000000000 --- a/imap_processing/packet_definitions/codice/lo_pha.xml +++ /dev/null @@ -1,141 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 276480 - - - - - - - CCSDS Packet Version Number (always 0) - - - CCSDS Packet Type Indicator (0=telemetry) - - - CCSDS Packet Secondary Header Flag (always 1) - - - CCSDS Packet Application Process ID - - - CCSDS Packet Grouping Flags (3=not part of group) - - - CCSDS Packet Sequence Count (increments with each new packet) - - - CCSDS Packet Length (number of bytes after Packet length minus 1) - - - CCSDS Packet Time Stamp (coarse time) - - - Spin Period reported by the Spacecraft - - - Acquisition Start Time (Seconds) - - - Acquisition Start Time (Subseconds) - - - Spare for alignment - - - Total number of TCR events - - - Total number of DCR events - - - Total number of APD-Only events - - - Number of events included in this packet - - - Whether the event data is compressed - - - Number of bytes in the Event Data array - - - Event Data - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/imap_processing/packet_definitions/codice/lo_sw_species_counts.xml b/imap_processing/packet_definitions/codice/lo_sw_species_counts.xml deleted file mode 100644 index eec5d4b8d..000000000 --- a/imap_processing/packet_definitions/codice/lo_sw_species_counts.xml +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 37748736 - - - - - - - CCSDS Packet Version Number (always 0) - - - CCSDS Packet Type Indicator (0=telemetry) - - - CCSDS Packet Secondary Header Flag (always 1) - - - CCSDS Packet Application Process ID - - - CCSDS Packet Grouping Flags (3=not part of group) - - - CCSDS Packet Sequence Count (increments with each new packet) - - - CCSDS Packet Length (number of bytes after Packet length minus 1) - - - CCSDS Packet Time Stamp (coarse time) - - - Spin Period reported by the Spacecraft - - - Acquisition Start Time (Seconds) - - - Acquisition Start Time (Subseconds) - - - Spare for alignment - - - Science Lookup Table Version/ID - - - Plan Table ID - - - Plan Step Number - - - View table used for data collapsing and compression - - - Half-spin when Reduced Gain Factor Operation was activated - - - Half-spin when No Scan Operation was activated - - - Bias Voltage Mode - - - Indicates a data quality issue - - - Compression Configuration - - - Number of bytes in the Data array - - - Data Array - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/imap_processing/packet_definitions/codice/omni_species_counts.xml b/imap_processing/packet_definitions/codice/omni_species_counts.xml deleted file mode 100644 index 9187c79b6..000000000 --- a/imap_processing/packet_definitions/codice/omni_species_counts.xml +++ /dev/null @@ -1,155 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - CCSDS Packet Version Number (always 0) - - - CCSDS Packet Type Indicator (0=telemetry) - - - CCSDS Packet Secondary Header Flag (always 1) - - - CCSDS Packet Application Process ID - - - CCSDS Packet Grouping Flags (3=not part of group) - - - CCSDS Packet Sequence Count (increments with each new packet) - - - CCSDS Packet Length (number of bytes after Packet length minus 1) - - - CCSDS Packet Time Stamp (coarse time) - - - Spin Period reported by the Spacecraft - - - Acquisition Start Time (Seconds) - - - Acquisition Start Time (Subseconds) - - - Spare for alignment - - - Science Lookup Table Version/ID - - - Plan Table ID - - - Plan Step Number - - - View table used for data collapsing and compression - - - Half-spin when Reduced Gain Factor Operation was activated - - - Half-spin when No Scan Operation was activated - - - Bias Voltage Mode - - - Indicates a data quality issue - - - Compression Configuration - - - Number of bytes in the Data array - - - Data Array - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/imap_processing/swe/decom_swe.py b/imap_processing/swe/decom_swe.py index 55dd46d0d..31960ca5a 100644 --- a/imap_processing/swe/decom_swe.py +++ b/imap_processing/swe/decom_swe.py @@ -1,4 +1,4 @@ -from imap_processing import decom, packet_definition_directory +from imap_processing import decom, imap_module_directory def decom_packets(packet_file: str): @@ -13,5 +13,7 @@ def decom_packets(packet_file: str): List List of all the unpacked data """ - xtce_document = f"{packet_definition_directory}/swe_packet_definition.xml" + xtce_document = ( + f"{imap_module_directory}/swe/packet_definitions/swe_packet_definition.xml" + ) return decom.decom_packets(packet_file, xtce_document) diff --git a/imap_processing/swe/packet_definitions/__init__.py b/imap_processing/swe/packet_definitions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/imap_processing/packet_definitions/swe_packet_definition.xml b/imap_processing/swe/packet_definitions/swe_packet_definition.xml similarity index 100% rename from imap_processing/packet_definitions/swe_packet_definition.xml rename to imap_processing/swe/packet_definitions/swe_packet_definition.xml diff --git a/poetry.lock b/poetry.lock index eaa67ffb4..842a0225b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. [[package]] name = "accessible-pygments" version = "0.0.4" description = "A collection of accessible pygments styles" +category = "main" optional = true python-versions = "*" files = [ @@ -18,6 +19,7 @@ pygments = ">=1.5" name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -29,6 +31,7 @@ files = [ name = "atomicwrites" version = "1.4.1" description = "Atomic file writes." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -39,6 +42,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -57,6 +61,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.12.1" description = "Internationalization utilities" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -68,6 +73,7 @@ files = [ name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" +category = "main" optional = true python-versions = ">=3.6.0" files = [ @@ -86,6 +92,7 @@ lxml = ["lxml"] name = "bitarray" version = "2.8.1" description = "efficient arrays of booleans -- C extension" +category = "main" optional = false python-versions = "*" files = [ @@ -197,6 +204,7 @@ files = [ name = "bitstring" version = "4.1.1" description = "Simple construction, analysis and modification of binary data." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -211,6 +219,7 @@ bitarray = ">=2.8.0,<3.0.0" name = "black" version = "23.7.0" description = "The uncompromising code formatter." +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -257,6 +266,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -268,6 +278,7 @@ files = [ name = "cfgv" version = "3.4.0" description = "Validate configuration and produce human readable error messages." +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -279,6 +290,7 @@ files = [ name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -363,6 +375,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -377,6 +390,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -388,6 +402,7 @@ files = [ name = "coverage" version = "7.3.0" description = "Code coverage measurement for Python" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -455,6 +470,7 @@ toml = ["tomli"] name = "distlib" version = "0.3.7" description = "Distribution utilities" +category = "main" optional = true python-versions = "*" files = [ @@ -466,6 +482,7 @@ files = [ name = "docutils" version = "0.20.1" description = "Docutils -- Python Documentation Utilities" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -477,6 +494,7 @@ files = [ name = "et-xmlfile" version = "1.1.0" description = "An implementation of lxml.xmlfile for the standard library" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -488,6 +506,7 @@ files = [ name = "filelock" version = "3.12.3" description = "A platform independent file lock." +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -506,6 +525,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pyt name = "identify" version = "2.5.27" description = "File identification library for Python" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -520,6 +540,7 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -531,6 +552,7 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -542,6 +564,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -561,6 +584,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -572,6 +596,7 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -589,6 +614,7 @@ i18n = ["Babel (>=2.7)"] name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -613,6 +639,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -672,6 +699,7 @@ files = [ name = "mdit-py-plugins" version = "0.4.0" description = "Collection of plugins for markdown-it-py" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -691,6 +719,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -702,6 +731,7 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -713,6 +743,7 @@ files = [ name = "myst-parser" version = "2.0.0" description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -739,6 +770,7 @@ testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4, name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" +category = "main" optional = true python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -753,6 +785,7 @@ setuptools = "*" name = "numpy" version = "1.25.2" description = "Fundamental package for array computing in Python" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -787,6 +820,7 @@ files = [ name = "openpyxl" version = "3.1.2" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -801,6 +835,7 @@ et-xmlfile = "*" name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -812,6 +847,7 @@ files = [ name = "pandas" version = "2.1.0" description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -873,6 +909,7 @@ xml = ["lxml (>=4.8.0)"] name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -884,6 +921,7 @@ files = [ name = "platformdirs" version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -899,6 +937,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -914,6 +953,7 @@ testing = ["pytest", "pytest-benchmark"] name = "pre-commit" version = "3.4.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -932,6 +972,7 @@ virtualenv = ">=20.10.0" name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -943,6 +984,7 @@ files = [ name = "pydata-sphinx-theme" version = "0.13.3" description = "Bootstrap-based Sphinx theme from the PyData community" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -969,6 +1011,7 @@ test = ["codecov", "pytest", "pytest-cov", "pytest-regressions"] name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -983,6 +1026,7 @@ plugins = ["importlib-metadata"] name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1007,6 +1051,7 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1025,6 +1070,7 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1039,6 +1085,7 @@ six = ">=1.5" name = "pytz" version = "2023.3.post1" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -1050,6 +1097,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1058,7 +1106,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1066,15 +1113,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1091,7 +1131,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1099,7 +1138,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1109,6 +1147,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1130,6 +1169,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "ruff" version = "0.0.253" description = "An extremely fast Python linter, written in Rust." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1156,6 +1196,7 @@ files = [ name = "setuptools" version = "68.1.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1172,6 +1213,7 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1183,6 +1225,7 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +category = "main" optional = true python-versions = "*" files = [ @@ -1194,6 +1237,7 @@ files = [ name = "soupsieve" version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1205,6 +1249,7 @@ files = [ name = "space-packet-parser" version = "4.1.0" description = "A CCSDS telemetry packet decoding library based on the XTCE packet format description standard." +category = "main" optional = false python-versions = ">=3.8,<3.12" files = [ @@ -1219,6 +1264,7 @@ bitstring = ">=3.0.0,<4.2" name = "sphinx" version = "7.2.5" description = "Python documentation generator" +category = "main" optional = true python-versions = ">=3.9" files = [ @@ -1254,6 +1300,7 @@ test = ["cython (>=3.0)", "filelock", "html5lib", "pytest (>=4.6)", "setuptools name = "sphinxcontrib-applehelp" version = "1.0.7" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +category = "main" optional = true python-versions = ">=3.9" files = [ @@ -1272,6 +1319,7 @@ test = ["pytest"] name = "sphinxcontrib-devhelp" version = "1.0.5" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" +category = "main" optional = true python-versions = ">=3.9" files = [ @@ -1290,6 +1338,7 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.4" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +category = "main" optional = true python-versions = ">=3.9" files = [ @@ -1308,6 +1357,7 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -1322,6 +1372,7 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-qthelp" version = "1.0.6" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" +category = "main" optional = true python-versions = ">=3.9" files = [ @@ -1340,6 +1391,7 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.9" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" +category = "main" optional = true python-versions = ">=3.9" files = [ @@ -1358,6 +1410,7 @@ test = ["pytest"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = true python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1369,6 +1422,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1380,6 +1434,7 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1391,6 +1446,7 @@ files = [ name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" +category = "main" optional = false python-versions = ">=2" files = [ @@ -1402,6 +1458,7 @@ files = [ name = "urllib3" version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1419,6 +1476,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "virtualenv" version = "20.24.4" description = "Virtual Python Environment builder" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1439,6 +1497,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "xarray" version = "2023.8.0" description = "N-D labeled arrays and datasets in Python" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1463,6 +1522,7 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] name = "zipp" version = "3.16.2" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = true python-versions = ">=3.8" files = [ diff --git a/tools/xtce_generation/ccsds_header_xtce_generator.py b/tools/xtce_generation/ccsds_header_xtce_generator.py index 4add2117f..e9a53e135 100644 --- a/tools/xtce_generation/ccsds_header_xtce_generator.py +++ b/tools/xtce_generation/ccsds_header_xtce_generator.py @@ -3,44 +3,44 @@ def __init__(self): self.parameters = [ { "name": "VERSION", - "parameterTypeRef": "uint3", + "parameterTypeRef": "UINT3", "description": "CCSDS Packet Version Number (always 0)", }, { "name": "TYPE", - "parameterTypeRef": "uint1", + "parameterTypeRef": "UINT1", "description": "CCSDS Packet Type Indicator (0=telemetry)", }, { "name": "SEC_HDR_FLG", - "parameterTypeRef": "uint1", + "parameterTypeRef": "UINT1", "description": "CCSDS Packet Secondary Header Flag (always 1)", }, { "name": "PKT_APID", - "parameterTypeRef": "uint11", + "parameterTypeRef": "UINT11", "description": "CCSDS Packet Application Process ID", }, { "name": "SEG_FLGS", - "parameterTypeRef": "uint2", + "parameterTypeRef": "UINT2", "description": "CCSDS Packet Grouping Flags (3=not part of group)", }, { "name": "SRC_SEQ_CTR", - "parameterTypeRef": "uint14", + "parameterTypeRef": "UINT14", "description": "CCSDS Packet Sequence Count " "(increments with each new packet)", }, { "name": "PKT_LEN", - "parameterTypeRef": "uint16", + "parameterTypeRef": "UINT16", "description": "CCSDS Packet Length " "(number of bytes after Packet length minus 1)", }, { "name": "SHCOARSE", - "parameterTypeRef": "uint32", + "parameterTypeRef": "UINT32", "description": "CCSDS Packet Time Stamp (coarse time)", }, ] diff --git a/tools/xtce_generation/telemetry_generator.py b/tools/xtce_generation/telemetry_generator.py index e0a6df088..783147426 100644 --- a/tools/xtce_generation/telemetry_generator.py +++ b/tools/xtce_generation/telemetry_generator.py @@ -1,16 +1,23 @@ import xml.etree.ElementTree as Et from datetime import datetime +import pandas as pd + from tools.xtce_generation.ccsds_header_xtce_generator import CCSDSParameters class TelemetryGenerator: - def __init__(self, packet_name, path_to_excel_file, apid, sci_byte=None, pkt=None): + def __init__(self, packet_name, path_to_excel_file, apid, pkt=None): self.packet_name = packet_name - self.path_to_excel_file = path_to_excel_file self.apid = apid - self.sci_byte = sci_byte - self.pkt = pkt + self.source_link = "http://www.omg.org/space/xtce" + + if pkt is None: + # Read excel sheet + xls = pd.ExcelFile(path_to_excel_file) + self.pkt = xls.parse(packet_name) + else: + self.pkt = pkt def create_telemetry_xml(self): """ @@ -23,26 +30,26 @@ def create_telemetry_xml(self): - telemetry_metadata: The TelemetryMetaData element. """ # Register the XML namespace - Et.register_namespace("xtce", "http://www.omg.org/space") + Et.register_namespace("xtce", self.source_link) # Get the current date and time current_date = datetime.now().strftime("%Y-%m") # Create the root element and add namespaces - root = Et.Element("{http://www.omg.org/space}SpaceSystem") + root = Et.Element("xtce:SpaceSystem") + root.attrib["xmlns:xtce"] = f"{self.source_link}" root.attrib["name"] = self.packet_name + # xmlns:xtce="http://www.omg.org/space/xtce" # Create the Header element with attributes 'date', 'version', and 'author' # Versioning is used to keep track of changes to the XML file. - header = Et.SubElement(root, "{http://www.omg.org/space}Header") + header = Et.SubElement(root, "xtce:Header") header.attrib["date"] = current_date header.attrib["version"] = "1.0" header.attrib["author"] = "IMAP SDC" # Create the TelemetryMetaData element - telemetry_metadata = Et.SubElement( - root, "{http://www.omg.org/space}TelemetryMetaData" - ) + telemetry_metadata = Et.SubElement(root, "xtce:TelemetryMetaData") # Create the ParameterTypeSet element parameter_type_set = Et.SubElement(telemetry_metadata, "xtce:ParameterTypeSet") @@ -52,80 +59,73 @@ def create_telemetry_xml(self): return root, parameter_type_set, parameter_set, telemetry_metadata - def extract_data_info(self, mnemonic_names): - """ - Extract unique lengths from the 'lengthInBits' column and the - data types for the given mnemonic_names. - - Parameters: - - mnemonic_names: A list or single string of mnemonic names to - extract data types from - - Returns: - - unique_lengths: Unique values from the 'lengthInBits' column. - - data_types: a dictionary mapping the given mnemonic names to the - data types from the packet. + def get_unique_bits_length(self): + """Get unique values from the 'lengthInBits' column and create dictionary + with key and value using the dataType and lengthInBits. + Eg. + { + 'UINT16': 16, + 'UINT32': 32, + 'BYTE13000': 13000, + } + Returns + ------- + dict + dictionary containing all unique bits lengths """ # Extract unique values from the 'lengthInBits' column - unique_lengths = self.pkt["lengthInBits"].unique().astype(int) - - # Create data dictionary for different mnemonics - data_types = {} - - for name in mnemonic_names: - # This is pretty generic and might not work for some instruments. - data_type = self.pkt[self.pkt["mnemonic"].str.contains(name)]["dataType"] - if data_type.empty: - data_types[name] = "DefaultDataType" - else: - data_types[name] = data_type.values[0] - - return unique_lengths, data_types - - def create_parameter_types(self, parameter_type_set, unique_lengths, data_types): + length_in_bits = self.pkt["lengthInBits"] + data_types = self.pkt["dataType"] + unique_lengths = {} + for index in range(len(length_in_bits)): + # Here, we are creating a dictionary like this: + # { + # 'UINT16': 16, + # 'UINT32': 32, + # 'BYTE13000': 13000, + # } + unique_lengths[ + f"{data_types[index]}{length_in_bits[index]}" + ] = length_in_bits[index] + # Sort the dictionary by the value (lengthInBits) + unique_lengths = dict(sorted(unique_lengths.items(), key=lambda item: item[1])) + return unique_lengths + + def create_parameter_types(self, parameter_type_set, unique_lengths): """ Create parameter types based on 'dataType' for the unique 'lengthInBits' values. This will loop through the unique lengths and create a ParameterType element - for each length representing a UINT type. + for each length representing a data type. Parameters: - parameter_type_set: The ParameterTypeSet element where parameter types are. - unique_lengths: Unique values from the 'lengthInBits' column. - - data_types: a dictionary mapping the given mnemonic names to the - data types from the packet. Returns: - parameter_type_set: The updated ParameterTypeSet element. """ - for size in unique_lengths: - parameter_type_ref = f"uint{size}" # Use UINT for other sizes - - if size == self.sci_byte: - parameter_type_ref = ( - "BYTE" # Use BYTE type for both "Data" and "Event_Data" - ) - elif size == self.sci_byte: - parameter_type_ref = data_types[ - "Data" - ] # use dataType of "Data" in data_types dict - elif data_types["EventData"] and size == self.sci_byte: - # Use data_type_event_data for "Event_Data" mnemonic - parameter_type_ref = data_types["EventData"] - - parameter_type = Et.SubElement(parameter_type_set, "xtce:ParameterType") - parameter_type.attrib["name"] = f"uint{size}" - parameter_type.attrib["signed"] = "false" - - encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") - encoding.attrib["sizeInBits"] = str(size) - encoding.attrib["encoding"] = "unsigned" - - # Create BinaryParameterType if parameter_type_ref is "BYTE" - if parameter_type_ref == "BYTE": + for parameter_type_ref_name, size in unique_lengths.items(): + if "UINT" in parameter_type_ref_name: + parameter_type = Et.SubElement(parameter_type_set, "xtce:ParameterType") + parameter_type.attrib["name"] = parameter_type_ref_name + parameter_type.attrib["signed"] = "false" + + encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") + encoding.attrib["sizeInBits"] = str(size) + encoding.attrib["encoding"] = "unsigned" + elif "SINT" in parameter_type_ref_name: + parameter_type = Et.SubElement(parameter_type_set, "xtce:ParameterType") + parameter_type.attrib["name"] = parameter_type_ref_name + parameter_type.attrib["signed"] = "true" + + encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") + encoding.attrib["sizeInBits"] = str(size) + encoding.attrib["encoding"] = "signed" + elif "BYTE" in parameter_type_ref_name: binary_parameter_type = Et.SubElement( parameter_type_set, "xtce:BinaryParameterType" ) - binary_parameter_type.attrib["name"] = parameter_type_ref + binary_parameter_type.attrib["name"] = parameter_type_ref_name Et.SubElement(binary_parameter_type, "xtce:UnitSet") @@ -136,9 +136,7 @@ def create_parameter_types(self, parameter_type_set, unique_lengths, data_types) size_in_bits = Et.SubElement(binary_data_encoding, "xtce:SizeInBits") fixed_value = Et.SubElement(size_in_bits, "xtce:FixedValue") - fixed_value.text = str( - self.sci_byte - ) # Set the size in bits to sci_byte + fixed_value.text = str(size) return parameter_type_set @@ -164,16 +162,16 @@ def create_ccsds_packet_parameters(self, parameter_set, ccsds_parameters): return parameter_set def create_container_set( - self, telemetry_metadata, ccsds_parameters, apid, container_name + self, telemetry_metadata, ccsds_parameters, container_name ): """ Create XML elements for ContainerSet, CCSDSPacket SequenceContainer, - and SciencePacket SequenceContainer. + and Packet SequenceContainer. Parameters: - telemetry_metadata: The TelemetryMetaData element where containers are. - ccsds_parameters: A list of dictionaries containing CCSDS parameter data. - - APID: The APID value used for comparison in SciencePacket. + - container_name: The name of sequence container Returns: - telemetry_metadata: The updated TelemetryMetaData element. @@ -188,104 +186,87 @@ def create_container_set( ccsds_packet_container, "xtce:EntryList" ) + # Populate EntryList for CCSDSPacket SequenceContainer for parameter_data in ccsds_parameters: parameter_ref_entry = Et.SubElement( ccsds_packet_entry_list, "xtce:ParameterRefEntry" ) parameter_ref_entry.attrib["parameterRef"] = parameter_data["name"] - # Create CoDICESciencePacket SequenceContainer + # Create Packet SequenceContainer that use CCSDSPacket SequenceContainer + # as base container science_container = Et.SubElement(container_set, "xtce:SequenceContainer") science_container.attrib["name"] = container_name base_container = Et.SubElement(science_container, "xtce:BaseContainer") base_container.attrib["containerRef"] = "CCSDSPacket" + # Add RestrictionCriteria element to use the given APID for comparison restriction_criteria = Et.SubElement(base_container, "xtce:RestrictionCriteria") comparison = Et.SubElement(restriction_criteria, "xtce:Comparison") comparison.attrib["parameterRef"] = "PKT_APID" - comparison.attrib["value"] = apid + comparison.attrib["value"] = f"{self.apid}" comparison.attrib["useCalibratedValue"] = "false" - Et.SubElement(science_container, "xtce:EntryList") - - return telemetry_metadata - - def add_science_parameters(self, science_container, pkt): - """ - Add ParameterRefEntry elements for SciencePacket. - Parameters: - - science_container: The SciencePacket SequenceContainer element. - - pkt: The DataFrame containing packet data. - Returns: - - codice_science_container: The updated SciencePacket - SequenceContainer element. - """ - # Get the 'mnemonic' values starting from row 10 - parameter_refs = pkt.loc[9:, "mnemonic"].tolist() + # Populate EntryList for packet SequenceContainer + packet_entry_list = Et.SubElement(science_container, "xtce:EntryList") + parameter_refs = self.pkt.loc[8:, "mnemonic"].tolist() for parameter_ref in parameter_refs: parameter_ref_entry = Et.SubElement( - science_container, "xtce:ParameterRefEntry" + packet_entry_list, "xtce:ParameterRefEntry" ) parameter_ref_entry.attrib["parameterRef"] = parameter_ref - return science_container + return telemetry_metadata - def create_parameters_from_dataframe(self, parameter_set, pkt): + def create_remaining_parameters(self, parameter_set): """ - Create XML elements for parameters based on DataFrame rows starting from row 10. + Create XML elements for parameters based on DataFrame rows starting + from SHCOARSE (also known as MET). Parameters: - parameter_set: The ParameterSet element where parameters will be added. - - pkt: The DataFrame containing packet data. Returns: - parameter_set: The updated ParameterSet element. """ - # Process rows from 10 until the last available row in the DataFrame - for index, row in pkt.iterrows(): - if index < 9: - continue # Skip rows before row 10 + # Process rows from SHCOARSE until the last available row in the DataFrame + for index, row in self.pkt.iterrows(): + if index < 8: + # Skip rows until SHCOARSE(also known as MET) which are + # part of CCSDS header + continue - parameter = Et.SubElement( - parameter_set, "{http://www.omg.org/space}Parameter" - ) + parameter = Et.SubElement(parameter_set, "xtce:Parameter") parameter.attrib["name"] = row["mnemonic"] - parameter_type_ref = "" - - # Use BYTE type for "Data" and "Event_Data" mnemonics - if row["mnemonic"] in ["Data", "Event_Data"]: - parameter_type_ref = "BYTE" - else: - parameter_type_ref = ( - f"uint{int(row['lengthInBits'])}" # Use UINT for others - ) + parameter_type_ref = f"{row['dataType']}{row['lengthInBits']}" parameter.attrib["parameterTypeRef"] = parameter_type_ref - description = Et.SubElement( - parameter, "{http://www.omg.org/space}LongDescription" - ) - description.text = row["shortDescription"] + description = Et.SubElement(parameter, "xtce:LongDescription") + + if row["longDescription"] is None and row["shortDescription"] is None: + description.text = "" + elif row["shortDescription"]: + description.text = row["shortDescription"] + else: + description.text = row["longDescription"] return parameter_set - def generate_telemetry_xml( - self, output_xml_path, mnemonic_names, science_packet_name="SciencePacket" - ): + def generate_telemetry_xml(self, output_xml_path, container_name): """ Create and output an XTCE file based on the data within the class. Parameters ---------- output_xml_path: the path for the final xml file - mnemonic_names: a list or single name for mnemonics for extracting data types - science_packet_name: The name of the science packet in the output xml + container_name: The name of the sequence container in the output xml """ - # Extract data information for "Data" and "Event_Data" mnemonics separately - (unique_lengths_data, data_types) = self.extract_data_info(mnemonic_names) - # Call the functions in order to generate the XML + unique_bits_lengths_data = self.get_unique_bits_length() + + # Here, we create the XML components so that we can add data in following steps ( telemetry_xml_root, parameter_type_set, @@ -293,26 +274,24 @@ def generate_telemetry_xml( telemetry_metadata, ) = self.create_telemetry_xml() - # Create parameter types for both "Data" and "Event_Data" - # Although this variable is not used, create_parameter_types has a side effect parameter_type_set = self.create_parameter_types( - parameter_type_set, - unique_lengths_data, - data_types, + parameter_type_set, unique_bits_lengths_data ) + # Create CCSDSPacket parameters and add them to the ParameterSet element parameter_set = self.create_ccsds_packet_parameters( parameter_set, CCSDSParameters().parameters ) + # Add remaining parameters to the ParameterSet element + parameter_set = self.create_remaining_parameters(parameter_set) + + # Create ContainerSet with CCSDSPacket SequenceContainer and packet + # SequenceContainer telemetry_metadata = self.create_container_set( telemetry_metadata, CCSDSParameters().parameters, - self.apid, - science_packet_name, + container_name=container_name, ) - self.add_science_parameters(telemetry_metadata, self.pkt) - - parameter_set = self.create_parameters_from_dataframe(parameter_set, self.pkt) # Create the XML tree and save the document tree = Et.ElementTree(telemetry_xml_root) diff --git a/tools/xtce_generation/xtce_generator_template.py b/tools/xtce_generation/xtce_generator_template.py new file mode 100644 index 000000000..3843f81db --- /dev/null +++ b/tools/xtce_generation/xtce_generator_template.py @@ -0,0 +1,50 @@ +from pathlib import Path + +from tools.xtce_generation.telemetry_generator import TelemetryGenerator + + +def main(): + """This function can be used by any instrument to generate XTCE + for certain number of packets. Change values where TODO is + """ + + # TODO: change instrument name + instrument_name = "codice" + current_directory = Path(__file__).parent + module_path = f"{current_directory}/../../imap_processing" + packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" + # TODO: Copy packet definition to tools/xtce_generation/ folder + # path_to_excel_file = f"{current_directory}/TLM_SWE_20230904.xlsx" + path_to_excel_file = f"{current_directory}/TLM_COD_20230629-110638.xlsx" + + # TODO: update packets dictionary with packet name and appId + + # SWE packets + # packets = { + # "P_SWE_APP_HK": 1330, + # "P_SWE_EVTMSG": 1317, + # "P_SWE_CEM_RAW": 1334, + # "P_SWE_SCIENCE": 1344, + # } + + # CoDICE packets + packets = { + "P_COD_HI_PHA": 1169, + "P_COD_LO_PHA": 1153, + "P_COD_LO_NSW_SPECIES_COUNTS": 1157, + "P_COD_HI_OMNI_SPECIES_COUNTS": 1172, + "P_COD_LO_SW_SPECIES_COUNTS": 1156, + } + + for packet_name, app_id in packets.items(): + print(packet_name) + telemetry_generator = TelemetryGenerator( + packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id + ) + telemetry_generator.generate_telemetry_xml( + f"{packet_definition_path}/{packet_name}.xml", packet_name + ) + + +if __name__ == "__main__": + main()