Merge pull request #1117 from debrief/develop

New release

HISTORY.rst

@@ -2,6 +2,13 @@
 History
 =======
 
+0.0.35 (2021-12-16)
+-------------------
+
+* Finalised WECDIS importer (with UX improvements)
+* Support new (VW/21) JChat format
+* Minor improvement to $PATH assignment in `install-Pepys`
+
 0.0.34 (2021-12-07)
 -------------------
 

bin/install_pepys.ps1

@@ -111,7 +111,7 @@ try {
     if (!($env:Path -split ';' -contains $pepys_bin_path)) {
         [Environment]::SetEnvironmentVariable(
             "PATH",
-            [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::User) + ";" + $pepys_bin_path,
+            $pepys_bin_path + ";" + [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::User),
             [EnvironmentVariableTarget]::User)
     }
 }

importers/jchat_importer.py

@@ -122,54 +122,43 @@ def _read_message_div(self, div, data_store, datafile, change_id):
             # Older format uses id=
             # Newer format uses msgid=
             msg_id = div.attrib["msgid"]  # Grabbing ID to help with error reporting
-            version = JCHAT_MODERN
         except KeyError:
             try:
                 msg_id = div.attrib["id"]
-                version = JCHAT_LEGACY
             except KeyError:
                 # Ignore any non-comment messages (e.g. connect/disconnect)
                 return
 
-        time_element = div.find("{*}tt/font")
-
         # Sample data included some "Marker" messages with the id="marker"
         if str.upper(msg_id) == "MARKER":
             return  # Ignore these messages
 
-        if time_element is None:
+        text_blocks = []
+        text_blocks.append([item for item in div.findall(".//*") if item.text])
+        if text_blocks[0]:
+            time_element = text_blocks[0][0]
+            platform_element = text_blocks[0][1]
+            msg_content_element = text_blocks[0][2:]
+
+        if not text_blocks[0] or len(text_blocks[0]) < 3:
             self.errors.append(
-                {self.error_type: f"Unable to read message {msg_id}. No timestamp provided"}
+                {
+                    self.error_type: f"Unable to read message {msg_id}. Not enough parts (expecting timestamp, platform, message)"
+                }
             )
             return
 
         time_string = time_element.text.strip("[").strip("]")
         timestamp = self.parse_timestamp(time_string, msg_id)
         time_element.record(self.name, "timestamp", timestamp)
 
-        if version == JCHAT_LEGACY:
-            platform_element = div.find("{*}b/a/font")
-        else:  # version == JCHAT_MODERN
-            platform_element = div.find("{*}b/font/a")
-
-        if platform_element is None:
-            self.errors.append(
-                {self.error_type: f"Unable to read message {msg_id}. No platform provided"}
-            )
-            return
         platform_quad = platform_element.text[0:4]
         platform_element.record(self.name, "platform", platform_quad)
         # Match on quadgraphs
         platform = self.get_cached_platform_from_quad(data_store, platform_quad, change_id)
 
-        msg_content_element = [element for element in div.iterfind("font")]
-
-        if not msg_content_element:
-            self.errors.append(
-                {self.error_type: f"Unable to read message {msg_id}. No message provided"}
-            )
-            return
         msg_content = self.parse_message_content(msg_content_element)
+
         if not msg_content:
             self.errors.append({self.error_type: f"Unable to parse JChat message {msg_id}."})
             return

importers/wecdis_importer.py

@@ -4,6 +4,7 @@
 from pepys_import.core.formats import unit_registry
 from pepys_import.core.formats.location import Location
 from pepys_import.core.validators import constants
+from pepys_import.file.highlighter.level import HighlightLevel
 from pepys_import.file.highlighter.support.combine import combine_tokens
 from pepys_import.file.importer import Importer
 from pepys_import.utils.unit_utils import convert_absolute_angle, convert_distance, convert_speed
@@ -19,6 +20,8 @@ class MsgType(str, Enum):
     TMA = "TMA"
     DEPTH = "PDS"
     TTM = "TTM"  # Could be TTM1, TTM2, TTM3
+    SPEED = "VEL"
+    HEADING = "HDG"
 
 
 class WecdisImporter(Importer):
@@ -27,12 +30,17 @@ def __init__(self):
             name="WECDIS File Format Importer",
             validation_level=constants.BASIC_LEVEL,
             short_name="WECDIS Importer",
+            default_privacy="Private",
             datafile_type="WECDIS",
         )
         self.current_line_no = None
         self.platform_name = None
+        # Potentially multiple sources for speed/heading
+        self.platform_speed = {}
+        self.platform_heading = {}
         self.timestamp = None
         self.elevation = None
+        self.set_highlighting_level(HighlightLevel.NONE)
 
     def can_load_this_type(self, suffix):
         return suffix.upper() == ".LOG" or suffix.upper() == ".TXT"
@@ -68,6 +76,10 @@ def _load_this_line(self, data_store, line_number, line, datafile, change_id):
                     self.handle_position(data_store, line_number, tokens, datafile, change_id)
             elif msg_type == MsgType.TMA:
                 self.handle_tma(data_store, line_number, tokens, datafile, change_id)
+            elif MsgType.SPEED in msg_type:
+                self.handle_speed(tokens, line_number)
+            elif MsgType.HEADING in msg_type:
+                self.handle_heading(tokens, line_number)
             # elif MsgType.TTM in msg_type:
             #     self.handle_ttm(data_store, line_number, tokens, datafile, change_id)
         datafile.flush_extracted_tokens()
@@ -107,6 +119,38 @@ def handle_depth(self, tokens, line_number):
                 self.elevation = -1 * depth
             depth_token.record(self.name, "Depth", self.elevation)
 
+    def handle_speed(self, tokens, line_number):
+        """Extracts the speed from the VEL line
+        :param tokens: A tokenised VEL line
+        :ptype tokens: Line (list of tokens)
+        :param line_number: The line number of the speed"""
+        speed_token = tokens[4]  # We're ignoring SPL
+        units = unit_registry.knot  # TODO - check whether we can derive units
+        if speed_token and speed_token.text:
+            speed_valid, speed = convert_speed(
+                speed_token.text, units, line_number, self.errors, self.error_type
+            )
+            if speed_valid:
+                key = "1" if tokens[1].text == MsgType.SPEED else tokens[1].text[-1]
+                self.platform_speed[key] = speed
+                speed_token.record(self.name, "Speed", self.platform_speed)
+
+    def handle_heading(self, tokens, line_number):
+        """Extracts the heading from the HDG line
+        :param tokens: A tokenised HDG line
+        :ptype tokens: Line (list of tokens)
+        :param line_number: The line number of the heading being parsed"""
+        heading_token = tokens[2]
+        # TODO - confirm we're always in degs
+        if heading_token and heading_token.text:
+            heading_valid, heading = convert_absolute_angle(
+                heading_token.text, line_number, self.errors, self.error_type
+            )
+            if heading_valid:
+                key = "1" if tokens[1].text == MsgType.HEADING else tokens[1].text[-1]
+                self.platform_heading[key] = heading
+                heading_token.record(self.name, "Heading", self.platform_heading)
+
     def handle_timestamp(self, dza_tokens, line_number):
         """Extracts the important information from a DZA (Timestamp) line
         :param dza_tokens: A tokenised DZA line
@@ -209,11 +253,15 @@ def handle_position(self, data_store, line_number, tokens, datafile, change_id):
         combine_tokens(source_token, sensor_token).record(self.name, "sensor", sensor_name)
 
         platform = self.get_cached_platform(data_store, self.platform_name, change_id=change_id)
-        sensor = self.get_cached_sensor(
-            data_store=data_store,
-            sensor_name=sensor_name,
-            sensor_type=sensor_token.text,
-            platform_id=platform.platform_id,
+        sensor_type = data_store.add_to_sensor_types("Location-Satellite", change_id=change_id).name
+        sensor = data_store.add_to_sensors(
+            name=sensor_token.text,
+            sensor_type=sensor_type,
+            host_id=platform.platform_id,
+            host_name=None,
+            host_nationality=None,
+            host_identifier=None,
+            privacy=self.default_privacy,
             change_id=change_id,
         )
 
@@ -240,7 +288,23 @@ def handle_position(self, data_store, line_number, tokens, datafile, change_id):
             if speed_valid:
                 state.speed = speed
                 speed_token.record(self.name, "speed", speed)
-
+        elif tokens[1].text.startswith("POS"):
+            key = "1" if tokens[1].text == MsgType.POSITION else tokens[1].text[-1]
+            # Speed / heading for POS is taken from the VEL/HDG lines
+            if self.platform_speed:
+                try:
+                    state.speed = self.platform_speed[key]
+                except KeyError:
+                    # Fall back to "1" if this position doesn't match
+                    if self.platform_speed["1"]:
+                        state.speed = self.platform_speed["1"]
+            if self.platform_heading:
+                try:
+                    state.heading = self.platform_heading[key]
+                except KeyError:
+                    # Fall back to "1" if this position doesn't match
+                    if self.platform_heading["1"]:
+                        state.heading = self.platform_heading["1"]
         if self.elevation:
             state.elevation = self.elevation
 
@@ -288,7 +352,7 @@ def handle_tma(self, data_store, line_number, tokens, datafile, change_id):
         detecting_platform = self.get_cached_platform(data_store, self.platform_name, change_id)
         detecting_sensor = self.get_cached_sensor(
             data_store,
-            sensor_name=None,  # Ask, as we don't know where it came from
+            sensor_name=f"{tokens[1].text}-{type_token.text}",  # Ask, as we don't know where it came from
             sensor_type=None,
             platform_id=detecting_platform.platform_id,
             change_id=change_id,
@@ -327,6 +391,10 @@ def handle_tma(self, data_store, line_number, tokens, datafile, change_id):
                 contact.orientation = course
                 course_token.record(self.name, "course", course)
 
+    # TODO - Extend the Wecdis importer to support TTM once we have a better idea
+    #     of what the TTM format actually is (below is based on MWC's REP parser
+    #     for Wecdis data)
+
     # def handle_ttm(self, data_store, line_number, tokens, datafile, change_id):
     #     """Handles TTM sensor contacts
     #     :param data_store: The data store that this is importing into

pepys_import/__init__.py

@@ -4,5 +4,5 @@
 
 __author__ = "Ian Mayo"
 __email__ = "ian@planetmayo.com"
-__version__ = "0.0.34"
+__version__ = "0.0.35"
 __build_timestamp__ = None

requirements.txt

@@ -1,4 +1,4 @@
-SQLAlchemy==1.4.*
+SQLAlchemy==1.4.27
 geoalchemy2>=0.9.4
 psycopg2==2.8.*
 pint==0.14

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.34
+current_version = 0.0.35
 commit = True
 tag = True
 

setup.py

@@ -44,6 +44,6 @@
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/debrief/pepys-import",
-    version="0.0.34",
+    version="0.0.35",
     zip_safe=False,
 )

tests/sample_data/jchat_files/combined_format.html

@@ -22,5 +22,11 @@
   <div id="34abc=34537">
     <tt><font>[08010809A]</font></tt><b><a href=""><font>ABCD_CMD</font></a></b><font><i>Legacy - font a swap - has i tag - no breaks</i></font>
   </div>
+  <div msgid="34bbb=34236">
+    <font><tt>[23112654A]</tt></font><font><b><a href="">SPLA_AB</a></font></b><font>Modern 2 - <br>no i tag<br>but has multiple<br>breaks</font>
+  </div>
+  <div msgid="34bbb=34235">
+    <font><tt>[06010709A]</tt></font><font><b><a href="">SPLB_XO</a></font></b><font>Modern 2 - no i tag - no breaks</font>
+  </div>
 </body>
 </html>

tests/sample_data/jchat_files/jchat_modern_2_format.html

@@ -0,0 +1,17 @@
+<html>
+<head>
+  <style type="text/css">
+    <!--
+      span.msgcontent { color: #0 }
+    -->
+  </style>
+</head>
+<body>
+  <div msgid="34bbb=34236">
+    <font><tt>[23112654A]</tt></font><font><b><a href="">SPLD_ME</a></font></b><font>Modern 2 - <br>no i tag<br>but has multiple<br>breaks</font>
+  </div>
+  <div msgid="34bbb=34235">
+    <font><tt>[06010709A]</tt></font><font><b><a href="">SPLD_XO</a></font></b><font>Modern 2 - no i tag - no breaks</font>
+  </div>
+</body>
+</html>

tests/sample_data/wecdis_files/multiple_pos_hdg_vel_sources.log

@@ -0,0 +1,23 @@
+$POSL,DZA,20211101,010230.123,012345678*21
+$POSL,SOUND,ON*10
+$POSL,CHART,LOAD,A:/File/Path,1234,SOURCE,1,01 Nov 2021, 2 Nov 2021*A01
+$POSL,VER,Some text,License,2020 Jun 08, S123*0D
+$POSL,VNM,NONSUCH*5C
+$POSL,PDS,11.1,M*00
+$POSL,POS,GPS,1234.1678,N,01234.1678,W,10.01,,Some text,N,,,,,*45
+$POSL,HDG,200.0,*CA
+$POSL,DZA,20211101,010233.010,012345678*18
+$POSL,POS,GPS,1234.1678,N,01234.1678,W,10.01,,Some text,N,,,,,*45
+$POSL,DZA,20211101,010305.010,012345678*18
+$POSL,HDG1,201.0,*CA
+$POSL,POS1,GPS,1235.0678,N,01236.1678,W,10.01,,Some text,N,,,,,*45
+$POSL,POS2,GPS2,1235.0678,N,01236.1678,W,10.01,,Some text,N,,,,,*45
+$POSL,PDS,19.7,M*00
+$POSL,HDG,201.2,*CA
+$POSL,HDG2,199.43
+$POSL,VEL,GPS,141.6,3.08,,,*11
+$POSL,VEL,SPL,,,7.2,0.0,7.2*12
+$POSL,VEL2,GPS,123,6.60,a,b,c,d*12
+$POSL,DZA,20211212,010335.010,012345678*18
+$POSL,POS1,GPS,1235.9978,N,01237.1678,W,10.01,,Some text,N,,,,,*45
+$POSL,POS2,GPS2,1235.9978,N,01237.1678,W,10.01,,Some text,N,,,,,*45

tests/sample_data/wecdis_files/wecdis_sample.log

@@ -6,6 +6,7 @@ $POSL,VNM,NONSUCH*5C
 $POSL,PDS,11.1,M*00
 $POSL,POS,GPS,1234.1678,N,01234.1678,W,10.01,,Some text,N,,,,,*45
 $POSL,CPOS,ABC_XY,1234.1678,N,00056.85432,W,340.0,2.8,Some text*42
+$POSL,HDG,200.0,a,b,c,d*CA
 $POSL,TZS,UTC*4A
 $POSL,CONTACT,AB,CD,123,M02 12kts,3.2,,,20201101,010230.124,HU,SHUPS----,99999.2,M,1234.0000,N,01253.9723,E,O,,,*67
 $POSL,DZA,20211101,010233.010,012345678*18
@@ -17,6 +18,8 @@ $POSL,DZA,20211101,010305.010,012345678*18
 $POSL,POS,GPS,1235.0678,N,01236.1678,W,10.01,,Some text,N,,,,,*45
 $POSL,CPOS,ABC_XY,1238.1678,N,00058.85432,W,330.0,5,Some text*42
 $POSL,PDS,19.7,M*00
+$POSL,HDG,201.2,*CA
+$POSL,VEL,GPS,141.6,3.08,,,*11
 $POSL,CONTACT,AB,CD,180,M02 12kts,12.5,,,20201101,010306.022,HU,SHUPS----,99999.2,M,1234.0000,N,01225.9723,E,O,,,*67
 $POSL,DZA,20211212,010335.010,012345678*18
 $POSL,POS,GPS,1235.9978,N,01237.1678,W,10.01,,Some text,N,,,,,*45