add parser for SPC Watch Probabilities #595

CHANGELOG.md

@@ -18,6 +18,7 @@ form of `process_messages_{a,b,e}` is now `TextProduct,str`.
 - Add option to `mcalc_feelslike` to support `mask_undefined`.
 - Add `twitter_media` link for generic text products that have a polygon (#586).
 - Add `limit_by_doy` option to `windrose_utils` to allow a day of year limit.
+- Add parser for SPC Watch Probabilities (WWP) product (#595).
 - Allow `pyiem.nws.nwsli` instance to be subscriptable for iterop.
 - Support passing `linewidths` to `MapPlot.contourf`.
 

data/product_examples/WWP/WWP9.txt

@@ -0,0 +1,32 @@
+551 
+WWUS40 KWNS 292158
+WWP9  
+
+TORNADO WATCH PROBABILITIES FOR WT 0159
+NWS STORM PREDICTION CENTER NORMAN OK
+0455 PM CDT FRI APR 29 2022
+
+WT 0159
+PROBABILITY TABLE:
+PROB OF 2 OR MORE TORNADOES                        :  60%
+PROB OF 1 OR MORE STRONG /EF2-EF5/ TORNADOES       :  50%
+PROB OF 10 OR MORE SEVERE WIND EVENTS              :  50%
+PROB OF 1 OR MORE WIND EVENTS >= 65 KNOTS          :  30%
+PROB OF 10 OR MORE SEVERE HAIL EVENTS              :  60%
+PROB OF 1 OR MORE HAIL EVENTS >= 2 INCHES          :  60%
+PROB OF 6 OR MORE COMBINED SEVERE HAIL/WIND EVENTS :  90%
+
+&&
+ATTRIBUTE TABLE:
+MAX HAIL /INCHES/                            : 5.0
+MAX WIND GUSTS SURFACE /KNOTS/               : 65
+MAX TOPS /X 100 FEET/                        : 600
+MEAN STORM MOTION VECTOR /DEGREES AND KNOTS/ : 24030
+PARTICULARLY DANGEROUS SITUATION             : NO
+
+&&
+FOR A COMPLETE GEOGRAPHICAL DEPICTION OF THE WATCH AND
+WATCH EXPIRATION INFORMATION SEE WOUS64 FOR WOU9.
+
+$$
+

data/product_examples/WWP/WWP_2006.txt

@@ -0,0 +1,30 @@
+000
+WWUS40 KWNS 252007
+WWP9  
+
+SEVERE THUNDERSTORM WATCH PROBABILITIES FOR WS 0249
+NWS STORM PREDICTION CENTER NORMAN OK
+0306 PM CDT TUE APR 25 2006
+
+WS 0249
+PROBABILITY TABLE:
+PROB OF 2 OR MORE TORNADOES                        : <05%
+PROB OF 1 OR MORE STRONG /F2-F5/ TORNADOES         : <02%
+PROB OF 10 OR MORE SEVERE WIND EVENTS              :  50%
+PROB OF 1 OR MORE WIND EVENTS >= 65 KNOTS          :  30%
+PROB OF 10 OR MORE SEVERE HAIL EVENTS              :  70%
+PROB OF 1 OR MORE HAIL EVENTS >= 2 INCHES          :  50%
+PROB OF 6 OR MORE COMBINED SEVERE HAIL/WIND EVENTS :  90%
+
+&&
+ATTRIBUTE TABLE:
+MAX HAIL /INCHES/                            : 2.5
+MAX WIND GUSTS SURFACE /KNOTS/               : 60
+MAX TOPS /X 100 FEET/                        : 500
+MEAN STORM MOTION VECTOR /DEGREES AND KNOTS/ : 31025
+PARTICULARLY DANGEROUS SITUATION             : NO
+
+&&
+FOR A COMPLETE GEOGRAPHICAL DEPICTION OF THE WATCH AND
+WATCH EXPIRATION INFORMATION SEE WOUS64 FOR WOU9.
+

data/product_examples/WWP/WWP_TEST.txt

@@ -0,0 +1,31 @@
+729 
+WWUS40 KWNS 271449
+WWP9  
+
+TEST...SEVERE THUNDERSTORM WATCH PROBABILITIES FOR WS 9999...TEST
+NWS STORM PREDICTION CENTER NORMAN OK
+0847 AM CST MON JAN 27 2020
+
+WS 9999
+PROBABILITY TABLE:
+PROB OF 2 OR MORE TORNADOES                        :  00%
+PROB OF 1 OR MORE STRONG /EF2-EF5/ TORNADOES       :  00%
+PROB OF 10 OR MORE SEVERE WIND EVENTS              :  00%
+PROB OF 1 OR MORE WIND EVENTS >= 65 KNOTS          :  00%
+PROB OF 10 OR MORE SEVERE HAIL EVENTS              :  00%
+PROB OF 1 OR MORE HAIL EVENTS >= 2 INCHES          :  00%
+PROB OF 6 OR MORE COMBINED SEVERE HAIL/WIND EVENTS :  00%
+
+&&
+ATTRIBUTE TABLE:
+MAX HAIL /INCHES/                            : 0.5
+MAX WIND GUSTS SURFACE /KNOTS/               : 50
+MAX TOPS /X 100 FEET/                        : 500
+MEAN STORM MOTION VECTOR /DEGREES AND KNOTS/ : 24035
+PARTICULARLY DANGEROUS SITUATION             : NO
+
+&&
+FOR A COMPLETE GEOGRAPHICAL DEPICTION OF THE WATCH AND
+WATCH EXPIRATION INFORMATION SEE WOUS64 FOR WOU9.
+
+$$

src/pyiem/models/wwp.py

@@ -0,0 +1,25 @@
+"""Pydantic data model for SPC Watch Probabilities (WWP)."""
+# pylint: disable=too-few-public-methods
+
+# third party
+from pydantic import BaseModel, Field
+
+
+class WWPModel(BaseModel):
+    """SPC Watch Probability."""
+
+    typ: str = Field(..., description="Type of watch")
+    num: int = Field(..., description="Watch number for the year")
+    tornadoes_2m: int = Field(None, description="Tornadoes 2m")
+    tornadoes_1m_strong: int = Field(None, description="Tornadoes 1m strong")
+    wind_10m: int = Field(None, description="Wind 10m")
+    wind_1m_65kt: int = Field(None, description="Wind 1m 65kt")
+    hail_10m: int = Field(None, description="Hail 10m")
+    hail_1m_2inch: int = Field(None, description="Hail 1m 2inch")
+    hail_wind_6m: int = Field(None, description="Hail wind 6m")
+    max_hail_size: float = Field(None, description="Max hail size")
+    max_wind_gust_knots: int = Field(None, description="Max wind gust knots")
+    max_tops_feet: int = Field(None, description="Max tops feet")
+    storm_motion_drct: int = Field(None, description="Storm motion drct")
+    storm_motion_sknt: int = Field(None, description="Storm motion sknt")
+    is_pds: bool = Field(..., description="Is PDS")

src/pyiem/nws/products/saw.py

@@ -71,17 +71,24 @@ def sql(self, txn):
           (psycopg2.transaction): a database transaction
         """
         if self.action == self.ISSUES:
-            # Delete any current entries
+            # Ensure we have a watch to update
             txn.execute(
-                "DELETE from watches WHERE num = %s and "
-                "extract(year from issued) = %s",
+                "select 1 from watches WHERE num = %s and "
+                "extract(year from issued at time zone 'UTC') = %s",
                 (self.ww_num, self.sts.year),
             )
+            if txn.rowcount == 0:
+                txn.execute(
+                    "INSERT into watches (num, issued) VALUES (%s, %s)",
+                    (self.ww_num, self.sts),
+                )
             # Insert into the main watches table
             giswkt = f"SRID=4326;{MultiPolygon([self.geometry]).wkt}"
             sql = (
-                "INSERT into watches (sel, issued, expired, type, report, "
-                "geom, num) VALUES(%s,%s,%s,%s,%s,%s,%s)"
+                "UPDATE watches SET sel = %s, issued = %s, expired = %s, "
+                "type = %s, report = %s, geom = %s, product_id_saw = %s "
+                "WHERE num = %s and "
+                "extract(year from issued at time zone 'UTC') = %s"
             )
             args = (
                 f"SEL{self.saw}",
@@ -90,7 +97,9 @@ def sql(self, txn):
                 DBTYPES[self.ww_type],
                 self.unixtext,
                 giswkt,
+                self.get_product_id(),
                 self.ww_num,
+                self.sts.year,
             )
             txn.execute(sql, args)
             # Update the watches_current table

src/pyiem/nws/products/wwp.py

@@ -0,0 +1,148 @@
+"""Parsing of Storm Prediction Center WWP Product."""
+import re
+
+from pyiem.models.wwp import WWPModel
+from pyiem.nws.product import TextProduct
+
+
+# The product format has been remarkably consistent over 16+ years!
+WS_RE = re.compile(r"W(?P<typ>[ST])\s+(?P<num>\d\d\d\d)\s*P?D?S?\n")
+PROB_RE = re.compile(
+    r"PROB OF 2 OR MORE TORNADOES\s+:\s+(?P<tornadoes_2m>[\<\>\d]+)%\n"
+    r"PROB OF 1 OR MORE STRONG /E?F2-E?F5/ TORNADOES\s+:"
+    r"\s+(?P<tornadoes_1m_strong>[\<\>\d]+)%\n"
+    r"PROB OF 10 OR MORE SEVERE WIND EVENTS\s+:\s+(?P<wind_10m>[\<\>\d]+)%\n"
+    r"PROB OF 1 OR MORE WIND EVENTS >= 65 KNOTS\s+:"
+    r"\s+(?P<wind_1m_65kt>[\<\>\d]+)%\n"
+    r"PROB OF 10 OR MORE SEVERE HAIL EVENTS\s+:\s+(?P<hail_10m>[\<\>\d]+)%\n"
+    r"PROB OF 1 OR MORE HAIL EVENTS >= 2 INCHES\s+:"
+    r"\s+(?P<hail_1m_2inch>[\<\>\d]+)%\n"
+    r"PROB OF 6 OR MORE COMBINED SEVERE HAIL/WIND EVENTS\s+:"
+    r"\s+(?P<wind_hail_6m>[\<\>\d]+)%\n"
+)
+ATTR_RE = re.compile(
+    r"MAX HAIL /INCHES/\s+:\s+(?P<max_hail_size>[\<\d\.]+)\n"
+    r"MAX WIND GUSTS SURFACE /KNOTS/\s+:\s+(?P<max_wind_gust_knots>[\<\d]+)\n"
+    r"MAX TOPS /X 100 FEET/\s+:\s+(?P<tops>\d*)\n"
+    r"MEAN STORM MOTION VECTOR /DEGREES AND KNOTS/\s+:"
+    r"\s+(?P<drct>\d\d\d)(?P<sknt>\d\d)\n"
+    r"PARTICULARLY DANGEROUS SITUATION\s+:\s+(?P<is_pds>NO|YES)"
+)
+
+
+def _convprob(val):
+    """Safe conversion."""
+    # appears currently that these values are always static
+    return int(val.replace(">", "").replace("<", ""))
+
+
+def _parse_data(tp):
+    """Fill out the data model."""
+    ws = WS_RE.search(tp.unixtext).groupdict()
+    prob = PROB_RE.search(tp.unixtext).groupdict()
+    attr = ATTR_RE.search(tp.unixtext).groupdict()
+    return WWPModel(
+        typ="TOR" if ws["typ"] == "T" else "SVR",
+        num=int(ws["num"]),
+        tornadoes_2m=_convprob(prob["tornadoes_2m"]),
+        tornadoes_1m_strong=_convprob(prob["tornadoes_1m_strong"]),
+        wind_10m=_convprob(prob["wind_10m"]),
+        wind_1m_65kt=_convprob(prob["wind_1m_65kt"]),
+        hail_10m=_convprob(prob["hail_10m"]),
+        hail_1m_2inch=_convprob(prob["hail_1m_2inch"]),
+        hail_wind_6m=_convprob(prob["wind_hail_6m"]),
+        max_hail_size=float(attr["max_hail_size"]),
+        max_wind_gust_knots=int(attr["max_wind_gust_knots"]),
+        max_tops_feet=int(attr["tops"]) * 100.0,
+        storm_motion_drct=int(attr["drct"]),
+        storm_motion_sknt=int(attr["sknt"]),
+        is_pds=(attr["is_pds"] == "YES"),
+    )
+
+
+class WWPProduct(TextProduct):
+    """Class representing a WWP Product"""
+
+    def __init__(self, text, utcnow=None):
+        """Constructor
+
+        Args:
+          text (str): text to parse
+        """
+        TextProduct.__init__(self, text, utcnow=utcnow)
+        self.data = _parse_data(self)
+
+    def is_test(self):
+        """Is this a test product?"""
+        return self.data.num > 9000 or self.unixtext.find("...TEST") > 0
+
+    def sql(self, txn):
+        """Do the necessary database work
+
+        Args:
+          (psycopg2.transaction): a database transaction
+        """
+        # First, check to see if we already have this num
+        txn.execute(
+            "SELECT num from watches where "
+            "extract(year from issued at time zone 'UTC') = %s and num = %s "
+            "and type = %s",
+            (self.valid.year, self.data.num, self.data.typ),
+        )
+        if txn.rowcount == 0:
+            # Insert an entry
+            txn.execute(
+                "INSERT into watches (num, issued, type) VALUES (%s, %s, %s)",
+                (self.data.num, self.valid, self.data.typ),
+            )
+        # Now, update the data
+        txn.execute(
+            "UPDATE watches SET "
+            "tornadoes_2m = %s, "
+            "tornadoes_1m_strong = %s, "
+            "wind_10m = %s, "
+            "wind_1m_65kt = %s, "
+            "hail_10m = %s, "
+            "hail_1m_2inch = %s, "
+            "hail_wind_6m = %s, "
+            "max_hail_size = %s, "
+            "max_wind_gust_knots = %s, "
+            "max_tops_feet = %s, "
+            "storm_motion_drct = %s, "
+            "storm_motion_sknt = %s, "
+            "is_pds = %s, "
+            "product_id_wwp = %s "
+            "WHERE extract(year from issued at time zone 'UTC') = %s "
+            "and num = %s",
+            (
+                self.data.tornadoes_2m,
+                self.data.tornadoes_1m_strong,
+                self.data.wind_10m,
+                self.data.wind_1m_65kt,
+                self.data.hail_10m,
+                self.data.hail_1m_2inch,
+                self.data.hail_wind_6m,
+                self.data.max_hail_size,
+                self.data.max_wind_gust_knots,
+                self.data.max_tops_feet,
+                self.data.storm_motion_drct,
+                self.data.storm_motion_sknt,
+                self.data.is_pds,
+                self.get_product_id(),
+                self.valid.year,
+                self.data.num,
+            ),
+        )
+
+
+def parser(text, utcnow=None):
+    """Parse SPC WWP Product.
+
+    Args:
+      text (str): the raw text to parse
+      utcnow (datetime): the current datetime with timezone set!
+
+    Returns:
+      WWPProduct instance
+    """
+    return WWPProduct(text, utcnow=utcnow)

tests/nws/products/test_wwp.py

@@ -0,0 +1,32 @@
+"""Can we process the WWP"""
+
+# Third party
+import pytest
+
+# Local
+from pyiem.nws.products.wwp import parser
+from pyiem.util import get_test_file
+
+
+def test_test_wwp():
+    """Test that we can handle test WWP products"""
+    prod = parser(get_test_file("WWP/WWP_TEST.txt"))
+    assert prod.is_test()
+
+
+@pytest.mark.parametrize("database", ["postgis"])
+def test_wwp9(dbcursor):
+    """Test that we can parse this."""
+    prod = parser(get_test_file("WWP/WWP9.txt"))
+    assert prod.data.num == 159
+    assert not prod.data.is_pds
+    prod.sql(dbcursor)
+
+
+@pytest.mark.parametrize("database", ["postgis"])
+def test_wwp2006(dbcursor):
+    """Test a WWP product from 2006."""
+    prod = parser(get_test_file("WWP/WWP_2006.txt"))
+    assert prod.data.num == 249
+    assert not prod.data.is_pds
+    prod.sql(dbcursor)