From 2221b77750e49c41a2952f55c8a03bc86f4db9d1 Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Fri, 25 Aug 2017 17:51:22 -0500 Subject: [PATCH 1/8] fix for usace swtwc data format change --- test/files/usace/swtwc/DSNT2.20170725.html | 168 +++++++++++++++++++++ test/files/usace/swtwc/DSNT2.current.html | 84 +++++++++++ test/files/usace/swtwc/MYST2.20130218.html | 84 ----------- test/files/usace/swtwc/MYST2.current.html | 84 ----------- test/usace_swtwc_test.py | 39 ++--- ulmo/usace/swtwc/core.py | 10 +- 6 files changed, 278 insertions(+), 191 deletions(-) create mode 100644 test/files/usace/swtwc/DSNT2.20170725.html create mode 100644 test/files/usace/swtwc/DSNT2.current.html delete mode 100644 test/files/usace/swtwc/MYST2.20130218.html delete mode 100644 test/files/usace/swtwc/MYST2.current.html diff --git a/test/files/usace/swtwc/DSNT2.20170725.html b/test/files/usace/swtwc/DSNT2.20170725.html new file mode 100644 index 00000000..ac1c1c16 --- /dev/null +++ b/test/files/usace/swtwc/DSNT2.20170725.html @@ -0,0 +1,168 @@ + + + + MYST2: Pat Mayse Lake (tabular) + + + +

MYST2: Pat Mayse Lake

+

Data processed at Tue Feb 19 15:07:26 2013 Central Standard Time

+
+ All information presented herein is the property of the government of the + United States of America and may be revised or deleted at any time without + prior notice. +
+ + + + DSNT2: Lake Texoma, Denison Dam (tabular) + + + +

DSNT2: Lake Texoma, Denison Dam

+

Data processed at Thu Jul 27 2017 22:26

+
+ All information presented herein is the property of the government of the + United States of America and may be revised or deleted at any time without + prior notice. +
+ Dissemination of this information without the expressed written consent of + the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is + strictly prohibited. +
+ These data are gathered electronically and, as presented, have not + undergone any form of quality control. These data are not guaranteed to be + valid or correct. +
+ Go to home page. +
+ Go to station list. +

+ Upstream stations: +

+ + Downstream station: + + +
+ Previous Day +
+ Next Day +
+ Current Day +
+ 
+    
+SHEF ID:       DSNT2     Lake Texoma, Denison Dam
+STATION TYPE:  Reservoir
+Normal Pool:  Bottom =   590.00, Top =   618.38
+
+               PRECIP PRECIP(A) ELEVATION   STORAGE   TW-ELEV    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
+                   in        in        ft     ac-ft        ft       cfs       cfs         F       deg       mph         %      W/m2      volt      volt
+              Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev  Regi-Rev  Regi-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev
+(US/Central)
+07/25 00:00      0.00      0.00    617.74   2573092    501.88       776        19     80.76    173.90      4.13     78.38      0.00     11.78     11.42
+07/25 01:00      0.00      0.00    617.73   2572324    501.87       564        19     80.10    135.90      1.42     81.98      0.00     11.77     11.37
+07/25 02:00      0.00      0.00    617.74   2573092    501.87       352        19     78.78     77.70      1.09     85.28      0.00     11.78     11.42
+07/25 03:00      0.00      0.00    617.74   2573092    501.87       352        19     77.13    112.20      3.82     90.53      0.00     11.78     10.60
+07/25 04:00      0.00      0.00    617.74   2573092    501.87       351        19     77.79    114.00      2.04     87.29      0.00     11.77     11.42
+07/25 05:00      0.00      0.00    617.74   2573092    501.87         2        19     76.24     87.10      2.25     91.50      0.00     11.77     11.42
+07/25 06:00      0.00      0.00    617.73   2572324    501.88         1        19     75.81    103.10      3.10     92.85      0.00     11.78     11.42
+07/25 07:00      0.00      0.00    617.73   2572324    501.87         0        19     75.25    105.10      3.42     93.64     17.00     11.78     11.42
+07/25 08:00      0.00      0.00    617.73   2572324    501.87         0        19     78.67    127.80      1.66     85.09    171.00     11.80     11.42
+07/25 09:00      0.00      0.00    617.73   2572324    501.88       111        19     81.52    198.90      3.80     77.40    389.00     11.81     11.42
+07/25 10:00      0.00      0.00    617.74   2573092    501.88       231        19     83.71    213.20      4.65     70.20    611.00     11.81     11.42
+07/25 11:00      0.00      0.00    617.71   2570787    503.68       259        19     85.26    225.40      4.55     65.80    782.00     11.81     11.42
+07/25 12:00      0.00      0.00    617.73   2572324    509.44       282      9819     88.00    158.40      3.38     62.32    705.00     11.80     11.42
+07/25 13:00      0.00      0.00    617.72   2571555    509.79       282     10569     91.08    148.60      2.87     59.33    985.00     11.79     11.42
+07/25 14:00      0.00      0.00    617.71   2570787    509.80       171     10319     91.42    149.90      4.06     53.60   1047.00     11.80     11.42
+07/25 15:00      0.00      0.00    617.69   2569250    509.79        76     10319     92.07    138.20      5.92     53.66    901.00     11.82     11.42
+07/25 16:00      0.00      0.00    617.67   2567713    509.79        73     10319     92.84    146.20      7.06     53.35    662.00     11.80     11.42
+07/25 17:00      0.00      0.00    617.65   2566177    509.79        74     10319     92.41    140.00      6.62     50.42    700.00     11.80     11.42
+07/25 18:00      0.00      0.00    617.64   2565408    509.80        74     10319     92.73    150.00      9.26     44.62    500.00     11.80     11.42
+07/25 19:00      0.00      0.00    617.64   2565408    509.79        74     10569     91.96    137.50      6.66     51.95    285.00     11.79     11.42
+07/25 20:00      0.00      0.00    617.62   2563871    509.79        50     10569     89.65    130.40      6.07     58.54     86.00     11.78     11.42
+07/25 21:00      0.00      0.00    617.61   2563103    509.81        25     10319     87.03    128.80      6.81     60.92      1.00     11.79     11.42
+07/25 22:00      0.00      0.00    617.64   2565408    508.62         0     10319     85.80    131.70      6.84     62.32      0.00     11.79     11.42
+07/25 23:00      0.00      0.00    617.61   2563103    502.61         0      1019     84.70    138.90      5.71     62.26      0.00     11.79     11.42
+
+
+
+ + + + Dissemination of this information without the expressed written consent of + the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is + strictly prohibited. +
+ These data are gathered electronically and, as presented, have not + undergone any form of quality control. These data are not guaranteed to be + valid or correct. +
+ Go to home page. +
+ Go to station list. +
+ Go to station map. +

+ + Downstream station: +

+ + Previous Day +
+ Next Day +
+ Current Day +

+ Display graphics. +
+ Display DCP transmissions. +

+ 
+    
+    SHEF ID:       MYST2     Pat Mayse Lake
+    STATION TYPE:  RESERVOIR
+    Flood Control Pool:  Bottom =  451.00, Top =  460.50
+    
+                   PRECIP PRECIP(2) ELEVATION   STORAGE    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
+                   INCHES    INCHES      FEET      ACFT       CFS       CFS     DEG-F   DEGREES       MPH   PERCENT      W/M2     VOLTS     VOLTS
+                     GOES      GOES      GOES      GOES  COMPRESQ  COMPRESQ      GOES      GOES      GOES      GOES      GOES      GOES      GOES
+       (Cen)
+    02/18 01:00      0.00      0.00    447.00     96402         9         0     55.20    173.00     18.02     56.37      0.00     12.99     12.83
+    02/18 02:00      0.00      0.00    446.99     96352         4         0     55.74    169.60     17.20     60.82      0.00     12.96     12.77
+    02/18 03:00      0.00      0.00    447.03     96556         0         0     55.29    170.20     17.79     64.79      0.00     12.94     12.77
+    02/18 04:00      0.00      0.00    447.00     96402         0         0     55.08    166.60     21.41     68.76      0.00     12.93     12.71
+    02/18 05:00      0.00      0.00    446.99     96352         0         0     55.96    166.00     19.19     72.00      0.00     12.92     12.71
+    02/18 06:00      0.00      0.00    446.99     96352         0         0     55.08    159.30     14.64     74.56      0.00     12.92     12.71
+    02/18 07:00      0.00      0.00    446.99     96352         0         0     55.40    154.30     13.20     77.19      0.00     12.91     12.71
+    02/18 08:00      0.00      0.00    446.99     96352        61         0     55.96    157.70     12.84     78.65     10.00     12.92     12.71
+    02/18 09:00      0.00      0.00    447.00     96402        61         0     57.96    171.90     16.03     78.05     32.00     13.10     12.71
+    02/18 10:00      0.00      0.00    447.00     96402       185         0     58.71    180.90     18.72     78.66     57.00     13.46     12.89
+    02/18 11:00      0.00      0.00    447.02     96505       188         0     62.56    198.90     23.01     73.84     79.00     13.68     13.24
+    02/18 12:00      0.00      0.00    447.03     96556       188         0     63.77    197.60     24.16     71.14    428.00     14.17     13.48
+    02/18 13:00      0.00      0.00    447.07     96762       128         0     66.29    214.80     29.18     58.32    662.00     13.73     13.94
+    02/18 14:00      0.00      0.00    447.07     96762       128         0     68.27    225.70     26.96     52.27    489.00     13.69     13.53
+    02/18 15:00      0.00      0.00    447.10     96917         3         0     69.69    273.90     23.58     31.94    575.00     13.76     13.48
+    02/18 16:00      0.00      0.00    447.01     96454         0         0     68.94    286.00     19.61     29.62    462.00     13.83     13.53
+    02/18 17:00      0.00      0.00    447.01     96454         0         0     63.77    312.20     19.79     32.12    264.00     13.86     13.59
+    02/18 18:00      0.00      0.00    446.98     96302         0         0     59.70    321.70     16.90     37.19     47.00     13.71     13.65
+    02/18 19:00      0.00      0.00    446.99     96352         0         0     56.17    343.30     11.96     42.25      0.00     13.26     13.53
+    02/18 20:00      0.00      0.00    446.98     96302         0         0     53.44    343.30     10.10     46.47      0.00     13.12     13.07
+    02/18 21:00      0.00      0.00    446.99     96352         0         0     50.90    338.00      6.26     50.50      0.00     13.08     12.89
+    02/18 22:00      0.00      0.00    446.98     96302         0         0     49.59    335.90      6.32     51.17      0.00     13.05     12.89
+    02/18 23:00      0.00      0.00    446.98     96302         0         0     47.50    342.30      2.99     52.46      0.00     13.03     12.89
+    02/19 00:00      0.00      0.00    446.98     96302         0         0     45.95    347.50      6.09     53.49      0.00     13.01     12.65
+    
+
+
+
+ + diff --git a/test/files/usace/swtwc/DSNT2.current.html b/test/files/usace/swtwc/DSNT2.current.html new file mode 100644 index 00000000..05710bca --- /dev/null +++ b/test/files/usace/swtwc/DSNT2.current.html @@ -0,0 +1,84 @@ + + + + + DSNT2: Lake Texoma, Denison Dam (tabular) + + + +

DSNT2: Lake Texoma, Denison Dam

+

Data processed at Fri Aug 25 2017 16:12

+
+ All information presented herein is the property of the government of the + United States of America and may be revised or deleted at any time without + prior notice. +
+ Dissemination of this information without the expressed written consent of + the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is + strictly prohibited. +
+ These data are gathered electronically and, as presented, have not + undergone any form of quality control. These data are not guaranteed to be + valid or correct. +
+ Go to home page. +
+ Go to station list. +

+ Upstream stations: +

+ + Downstream station: + + +
+ Previous Day +
+ Next Day +
+ Current Day +
+ 
+    
+SHEF ID:       DSNT2     Lake Texoma, Denison Dam
+STATION TYPE:  Reservoir
+Normal Pool:  Bottom =   590.00, Top =   616.85
+
+               PRECIP PRECIP(A) ELEVATION   STORAGE   TW-ELEV    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
+                   in        in        ft     ac-ft        ft       cfs       cfs         F       deg       mph         %      W/m2      volt      volt
+              Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev  Regi-Rev  Regi-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev
+(US/Central)
+08/25 00:00      0.00      0.00    617.90   2585386    508.67      5230     10069     75.60    109.60      6.01     82.04      0.00     13.73     13.18
+08/25 01:00      0.00      0.00    617.89   2584618    502.79      -901      -901     74.61    111.90      3.24     85.22      1.00     13.72     13.18
+08/25 02:00      0.00      0.00    617.89   2584618    502.00      -901      -901     73.27    108.20      5.65     86.56      0.00     13.72     13.18
+08/25 03:00      0.00      0.00    617.88   2583849    501.97      -901      -901     72.07     93.10      2.83     89.55      0.00     13.72     13.18
+08/25 04:00      0.00      0.00    617.88   2583849    501.97      -901      -901     71.96    110.90      4.25     87.29      0.00     13.72     13.18
+08/25 05:00      0.00      0.00    617.88   2583849    501.97      -901      -901     72.18    109.60      6.12     81.86      0.00     13.72     13.18
+08/25 06:00      0.00      0.00    617.86   2582313    504.50      -901      -901     71.96    107.90      6.39     77.95      1.00     13.72     13.18
+08/25 07:00      0.00      0.00    617.89   2584618    509.66      -901      -901     70.86    107.80      4.77     79.29      2.00     13.72     13.18
+08/25 08:00      0.00      0.00    617.88   2583849    509.88      -901      -901     70.65    109.60      6.14     77.83     64.00     13.72     13.18
+08/25 09:00      0.00      0.00    617.87   2583081    509.86      -901      -901     71.85    112.20      6.36     73.98    108.00     13.72     13.18
+08/25 10:00      0.00      0.00    617.85   2581544    509.88      -901      -901     74.39    108.80      8.57     67.27    320.00     13.72     13.18
+08/25 11:00      0.00      0.00    617.86   2582313    509.88      -901      -901     77.45    103.00      5.56     62.08    566.00     13.72     13.18
+08/25 12:00      0.00      0.00    617.85   2581544    509.87      -901      -901     79.21    112.40      6.97     60.55    524.00     13.72     13.18
+08/25 13:00      0.00      0.00    617.86   2582313    509.88      -901      -901     81.97    122.80      6.66     58.97   1128.00     13.72     13.18
+08/25 14:00      0.00      0.00    617.84   2580776    509.87      -901      -901     81.86    106.40      6.95     57.14    886.00     13.72     13.18
+08/25 15:00      0.00      0.00    617.84   2580776    509.90      -901      -901     83.39    123.70      7.21     56.65    723.00     13.72     13.18
+08/25 16:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 17:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 18:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 19:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 20:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 21:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 22:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 23:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+
+
+
+ + diff --git a/test/files/usace/swtwc/MYST2.20130218.html b/test/files/usace/swtwc/MYST2.20130218.html deleted file mode 100644 index d2cc696b..00000000 --- a/test/files/usace/swtwc/MYST2.20130218.html +++ /dev/null @@ -1,84 +0,0 @@ - - - - MYST2: Pat Mayse Lake (tabular) - - - -

MYST2: Pat Mayse Lake

-

Data processed at Tue Feb 19 15:07:26 2013 Central Standard Time

-
- All information presented herein is the property of the government of the - United States of America and may be revised or deleted at any time without - prior notice. -
- Dissemination of this information without the expressed written consent of - the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is - strictly prohibited. -
- These data are gathered electronically and, as presented, have not - undergone any form of quality control. These data are not guaranteed to be - valid or correct. -
- Go to home page. -
- Go to station list. -
- Go to station map. -

- - Downstream station: -

- - Previous Day -
- Next Day -
- Current Day -

- Display graphics. -
- Display DCP transmissions. -

- 
-    
-    SHEF ID:       MYST2     Pat Mayse Lake
-    STATION TYPE:  RESERVOIR
-    Flood Control Pool:  Bottom =  451.00, Top =  460.50
-    
-                   PRECIP PRECIP(2) ELEVATION   STORAGE    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
-                   INCHES    INCHES      FEET      ACFT       CFS       CFS     DEG-F   DEGREES       MPH   PERCENT      W/M2     VOLTS     VOLTS
-                     GOES      GOES      GOES      GOES  COMPRESQ  COMPRESQ      GOES      GOES      GOES      GOES      GOES      GOES      GOES
-       (Cen)
-    02/18 01:00      0.00      0.00    447.00     96402         9         0     55.20    173.00     18.02     56.37      0.00     12.99     12.83
-    02/18 02:00      0.00      0.00    446.99     96352         4         0     55.74    169.60     17.20     60.82      0.00     12.96     12.77
-    02/18 03:00      0.00      0.00    447.03     96556         0         0     55.29    170.20     17.79     64.79      0.00     12.94     12.77
-    02/18 04:00      0.00      0.00    447.00     96402         0         0     55.08    166.60     21.41     68.76      0.00     12.93     12.71
-    02/18 05:00      0.00      0.00    446.99     96352         0         0     55.96    166.00     19.19     72.00      0.00     12.92     12.71
-    02/18 06:00      0.00      0.00    446.99     96352         0         0     55.08    159.30     14.64     74.56      0.00     12.92     12.71
-    02/18 07:00      0.00      0.00    446.99     96352         0         0     55.40    154.30     13.20     77.19      0.00     12.91     12.71
-    02/18 08:00      0.00      0.00    446.99     96352        61         0     55.96    157.70     12.84     78.65     10.00     12.92     12.71
-    02/18 09:00      0.00      0.00    447.00     96402        61         0     57.96    171.90     16.03     78.05     32.00     13.10     12.71
-    02/18 10:00      0.00      0.00    447.00     96402       185         0     58.71    180.90     18.72     78.66     57.00     13.46     12.89
-    02/18 11:00      0.00      0.00    447.02     96505       188         0     62.56    198.90     23.01     73.84     79.00     13.68     13.24
-    02/18 12:00      0.00      0.00    447.03     96556       188         0     63.77    197.60     24.16     71.14    428.00     14.17     13.48
-    02/18 13:00      0.00      0.00    447.07     96762       128         0     66.29    214.80     29.18     58.32    662.00     13.73     13.94
-    02/18 14:00      0.00      0.00    447.07     96762       128         0     68.27    225.70     26.96     52.27    489.00     13.69     13.53
-    02/18 15:00      0.00      0.00    447.10     96917         3         0     69.69    273.90     23.58     31.94    575.00     13.76     13.48
-    02/18 16:00      0.00      0.00    447.01     96454         0         0     68.94    286.00     19.61     29.62    462.00     13.83     13.53
-    02/18 17:00      0.00      0.00    447.01     96454         0         0     63.77    312.20     19.79     32.12    264.00     13.86     13.59
-    02/18 18:00      0.00      0.00    446.98     96302         0         0     59.70    321.70     16.90     37.19     47.00     13.71     13.65
-    02/18 19:00      0.00      0.00    446.99     96352         0         0     56.17    343.30     11.96     42.25      0.00     13.26     13.53
-    02/18 20:00      0.00      0.00    446.98     96302         0         0     53.44    343.30     10.10     46.47      0.00     13.12     13.07
-    02/18 21:00      0.00      0.00    446.99     96352         0         0     50.90    338.00      6.26     50.50      0.00     13.08     12.89
-    02/18 22:00      0.00      0.00    446.98     96302         0         0     49.59    335.90      6.32     51.17      0.00     13.05     12.89
-    02/18 23:00      0.00      0.00    446.98     96302         0         0     47.50    342.30      2.99     52.46      0.00     13.03     12.89
-    02/19 00:00      0.00      0.00    446.98     96302         0         0     45.95    347.50      6.09     53.49      0.00     13.01     12.65
-    
-
-
-
- - diff --git a/test/files/usace/swtwc/MYST2.current.html b/test/files/usace/swtwc/MYST2.current.html deleted file mode 100644 index f79ecfca..00000000 --- a/test/files/usace/swtwc/MYST2.current.html +++ /dev/null @@ -1,84 +0,0 @@ - - - - MYST2: Pat Mayse Lake (tabular) - - - -

MYST2: Pat Mayse Lake

-

Data processed at Mon Mar 04 15:07:26 2013 Central Standard Time

-
- All information presented herein is the property of the government of the - United States of America and may be revised or deleted at any time without - prior notice. -
- Dissemination of this information without the expressed written consent of - the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is - strictly prohibited. -
- These data are gathered electronically and, as presented, have not - undergone any form of quality control. These data are not guaranteed to be - valid or correct. -
- Go to home page. -
- Go to station list. -
- Go to station map. -

- - Downstream station: -

- - Previous Day -
- Next Day -
- Current Day -

- Display graphics. -
- Display DCP transmissions. -

- 
-    
-    SHEF ID:       MYST2     Pat Mayse Lake
-    STATION TYPE:  RESERVOIR
-    Flood Control Pool:  Bottom =  451.00, Top =  460.50
-    
-                   PRECIP PRECIP(2) ELEVATION   STORAGE    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
-                   INCHES    INCHES      FEET      ACFT       CFS       CFS     DEG-F   DEGREES       MPH   PERCENT      W/M2     VOLTS     VOLTS
-                     GOES      GOES      GOES      GOES  COMPRESQ  COMPRESQ      GOES      GOES      GOES      GOES      GOES      GOES      GOES
-       (Cen)
-    03/04 01:00      0.00      0.00    446.84     95602         0         0     56.84    174.90     17.27     49.83      0.00     12.96     12.77
-    03/04 02:00      0.00      0.00    446.85     95652         0         0     56.28    172.70     18.93     51.60      0.00     12.93     12.71
-    03/04 03:00      0.00      0.00    446.85     95652         0         0     55.08    172.70     13.84     55.26      0.00     12.91     12.71
-    03/04 04:00      0.00      0.00    446.85     95652         0         0     54.86    177.40     12.90     60.88      0.00     12.90     12.71
-    03/04 05:00      0.00      0.00    446.85     95652         0         0     55.53    188.60     17.42     63.32      0.00     12.89     12.65
-    03/04 06:00      0.00      0.00    446.85     95652         0         0     55.40    188.60     22.32     66.99      0.00     12.88     12.65
-    03/04 07:00      0.00      0.00    446.84     95602         0         0     55.18    190.00     21.77     69.92      3.00     12.89     12.65
-    03/04 08:00      0.00      0.00    446.84     95602         0         0     55.74    189.80     21.15     71.63     44.00     13.13     12.65
-    03/04 09:00      0.00      0.00    446.86     95702         0         0     56.62    191.10     20.28     72.61     80.00     13.72     12.89
-    03/04 10:00      0.00      0.00    446.84     95602         0         0     58.37    195.70     21.02     69.19    170.00     14.12     13.53
-    03/04 11:00      0.00      0.00    446.85     95652         0         0     60.57    203.30     18.10     61.43    233.00     14.05     13.88
-    03/04 12:00      0.00      0.00    446.86     95702         0         0     64.44    209.00     20.82     51.36    747.00     13.69     13.83
-    03/04 13:00      0.00      0.00    446.89     95852         0         0     68.38    221.20     24.86     38.78    846.00     13.70     13.71
-    03/04 14:00      0.00      0.00    446.84     95602         0         0     70.03    212.20     23.34     36.33    744.00     13.70     13.83
-    03/04 15:00      0.00      0.00    446.87     95752      ----      ----     72.45    215.40     19.07     37.01    704.00     13.68     13.77
-    03/04 16:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 17:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 18:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 19:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 20:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 21:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 22:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 23:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/05 00:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    
-
-
-
- - diff --git a/test/usace_swtwc_test.py b/test/usace_swtwc_test.py index be2ee6a2..e79cb4aa 100644 --- a/test/usace_swtwc_test.py +++ b/test/usace_swtwc_test.py @@ -21,26 +21,27 @@ def test_get_stations(): def test_get_station_data(): test_station_data = [ - ('MYST2', '2013-02-18', { - 'code': 'MYST2', - 'description': 'Pat Mayse Lake', - 'station_type': 'RESERVOIR', - 'timezone': 'Cen', + ('DSNT2', '2017-07-25', { + 'code': 'DSNT2', + 'description': 'Lake Texoma, Denison Dam', + 'station_type': 'Reservoir', + 'timezone': 'US/Central', 'values': { - '2013-02-18 01:00:00': { - 'AIR-TEMP': 55.2, - 'BAT-LOAD': 12.83, - 'ELEVATION': 447.0, - 'INFLOW': 9.0, + '2017-07-25 01:00:00': { + 'AIR-TEMP': 80.10, + 'BAT-LOAD': 11.37, + 'ELEVATION': 617.73, + 'INFLOW': 564, 'PRECIP': 0.0, - 'PRECIP(2)': 0.0, - 'REL-HUMID': 56.37, - 'RELEASE': 0.0, + 'PRECIP(A)': 0.0, + 'REL-HUMID': 81.98, + 'RELEASE': 19, 'SOLAR-RAD': 0.0, - 'STORAGE': 96402.0, - 'VOLTAGE': 12.99, - 'WIND-DIR': 173.0, - 'WIND-SPEED': 18.02 + 'STORAGE': 2572324.0, + 'VOLTAGE': 11.77, + 'WIND-DIR': 135.90, + 'WIND-SPEED': 1.42, + 'TW-ELEV': 501.87 }, }, }), @@ -64,9 +65,9 @@ def test_get_station_data_current(): # can't easily test current since it is a moving target changes, but mostly # just make sure it parses correctl: current will have '---' values where # previous days do not - data_file = 'usace/swtwc/MYST2.current.html' + data_file = 'usace/swtwc/DSNT2.current.html' with test_util.mocked_urls(data_file): - station_data = ulmo.usace.swtwc.get_station_data('MYST2') + station_data = ulmo.usace.swtwc.get_station_data('DSNT2') assert len(station_data.get('values')) > 0 diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index cce1e1d4..63194f9e 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -68,6 +68,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): filename = '%s.%s.html'% (station_code, date_str) data_url = 'http://www.swt-wc.usace.army.mil/webdata/gagedata/' + filename + print data_url path = os.path.join(USACE_SWTWC_DIR, filename) with util.open_file_for_url(data_url, path) as f: @@ -90,6 +91,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): second_line = sio.readline() station_dict['station_type'] = second_line.strip().split(':')[1].strip() + print station_dict notes = [] @@ -103,9 +105,9 @@ def get_station_data(station_code, date=None, as_dataframe=False): if len(notes): station_dict['notes'] = '\n'.join(notes) - variable_names = _split_line(sio.readline()[15:], 10) - variable_units = _split_line(sio.readline()[15:], 10) - variable_sources = _split_line(sio.readline()[15:], 10) + variable_names = _split_line(sio.readline()[11:], 10) + variable_units = _split_line(sio.readline()[11:], 10) + variable_sources = _split_line(sio.readline()[11:], 10) station_dict['variables'] = dict([ (name, {'unit': unit, 'source': source}) @@ -115,7 +117,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): station_dict['timezone'] = sio.readline().strip().strip('()') column_names = ['datetime'] + variable_names - widths = [15] + ([10] * len(variable_names)) + widths = [14] + ([10] * len(variable_names)) converters = dict([ (variable_name, lambda x: float(x) if x != '----' else np.nan) for variable_name in variable_names From 7e0528b1c2bb10edcc82b35eac5b2867a17363ce Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Fri, 25 Aug 2017 18:02:30 -0500 Subject: [PATCH 2/8] remove debugging print stmt --- ulmo/usace/swtwc/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index 63194f9e..4c6e7cd4 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -68,7 +68,6 @@ def get_station_data(station_code, date=None, as_dataframe=False): filename = '%s.%s.html'% (station_code, date_str) data_url = 'http://www.swt-wc.usace.army.mil/webdata/gagedata/' + filename - print data_url path = os.path.join(USACE_SWTWC_DIR, filename) with util.open_file_for_url(data_url, path) as f: From 95e143153bb26dd349f0567d4890692bd5606b16 Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Fri, 25 Aug 2017 17:51:22 -0500 Subject: [PATCH 3/8] fix for usace swtwc data format change --- test/files/usace/swtwc/DSNT2.20170725.html | 168 +++++++++++++++++++++ test/files/usace/swtwc/DSNT2.current.html | 84 +++++++++++ test/files/usace/swtwc/MYST2.20130218.html | 84 ----------- test/files/usace/swtwc/MYST2.current.html | 84 ----------- test/usace_swtwc_test.py | 39 ++--- ulmo/usace/swtwc/core.py | 10 +- 6 files changed, 278 insertions(+), 191 deletions(-) create mode 100644 test/files/usace/swtwc/DSNT2.20170725.html create mode 100644 test/files/usace/swtwc/DSNT2.current.html delete mode 100644 test/files/usace/swtwc/MYST2.20130218.html delete mode 100644 test/files/usace/swtwc/MYST2.current.html diff --git a/test/files/usace/swtwc/DSNT2.20170725.html b/test/files/usace/swtwc/DSNT2.20170725.html new file mode 100644 index 00000000..ac1c1c16 --- /dev/null +++ b/test/files/usace/swtwc/DSNT2.20170725.html @@ -0,0 +1,168 @@ + + + + MYST2: Pat Mayse Lake (tabular) + + + +

MYST2: Pat Mayse Lake

+

Data processed at Tue Feb 19 15:07:26 2013 Central Standard Time

+
+ All information presented herein is the property of the government of the + United States of America and may be revised or deleted at any time without + prior notice. +
+ + + + DSNT2: Lake Texoma, Denison Dam (tabular) + + + +

DSNT2: Lake Texoma, Denison Dam

+

Data processed at Thu Jul 27 2017 22:26

+
+ All information presented herein is the property of the government of the + United States of America and may be revised or deleted at any time without + prior notice. +
+ Dissemination of this information without the expressed written consent of + the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is + strictly prohibited. +
+ These data are gathered electronically and, as presented, have not + undergone any form of quality control. These data are not guaranteed to be + valid or correct. +
+ Go to home page. +
+ Go to station list. +

+ Upstream stations: +

+ + Downstream station: + + +
+ Previous Day +
+ Next Day +
+ Current Day +
+ 
+    
+SHEF ID:       DSNT2     Lake Texoma, Denison Dam
+STATION TYPE:  Reservoir
+Normal Pool:  Bottom =   590.00, Top =   618.38
+
+               PRECIP PRECIP(A) ELEVATION   STORAGE   TW-ELEV    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
+                   in        in        ft     ac-ft        ft       cfs       cfs         F       deg       mph         %      W/m2      volt      volt
+              Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev  Regi-Rev  Regi-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev
+(US/Central)
+07/25 00:00      0.00      0.00    617.74   2573092    501.88       776        19     80.76    173.90      4.13     78.38      0.00     11.78     11.42
+07/25 01:00      0.00      0.00    617.73   2572324    501.87       564        19     80.10    135.90      1.42     81.98      0.00     11.77     11.37
+07/25 02:00      0.00      0.00    617.74   2573092    501.87       352        19     78.78     77.70      1.09     85.28      0.00     11.78     11.42
+07/25 03:00      0.00      0.00    617.74   2573092    501.87       352        19     77.13    112.20      3.82     90.53      0.00     11.78     10.60
+07/25 04:00      0.00      0.00    617.74   2573092    501.87       351        19     77.79    114.00      2.04     87.29      0.00     11.77     11.42
+07/25 05:00      0.00      0.00    617.74   2573092    501.87         2        19     76.24     87.10      2.25     91.50      0.00     11.77     11.42
+07/25 06:00      0.00      0.00    617.73   2572324    501.88         1        19     75.81    103.10      3.10     92.85      0.00     11.78     11.42
+07/25 07:00      0.00      0.00    617.73   2572324    501.87         0        19     75.25    105.10      3.42     93.64     17.00     11.78     11.42
+07/25 08:00      0.00      0.00    617.73   2572324    501.87         0        19     78.67    127.80      1.66     85.09    171.00     11.80     11.42
+07/25 09:00      0.00      0.00    617.73   2572324    501.88       111        19     81.52    198.90      3.80     77.40    389.00     11.81     11.42
+07/25 10:00      0.00      0.00    617.74   2573092    501.88       231        19     83.71    213.20      4.65     70.20    611.00     11.81     11.42
+07/25 11:00      0.00      0.00    617.71   2570787    503.68       259        19     85.26    225.40      4.55     65.80    782.00     11.81     11.42
+07/25 12:00      0.00      0.00    617.73   2572324    509.44       282      9819     88.00    158.40      3.38     62.32    705.00     11.80     11.42
+07/25 13:00      0.00      0.00    617.72   2571555    509.79       282     10569     91.08    148.60      2.87     59.33    985.00     11.79     11.42
+07/25 14:00      0.00      0.00    617.71   2570787    509.80       171     10319     91.42    149.90      4.06     53.60   1047.00     11.80     11.42
+07/25 15:00      0.00      0.00    617.69   2569250    509.79        76     10319     92.07    138.20      5.92     53.66    901.00     11.82     11.42
+07/25 16:00      0.00      0.00    617.67   2567713    509.79        73     10319     92.84    146.20      7.06     53.35    662.00     11.80     11.42
+07/25 17:00      0.00      0.00    617.65   2566177    509.79        74     10319     92.41    140.00      6.62     50.42    700.00     11.80     11.42
+07/25 18:00      0.00      0.00    617.64   2565408    509.80        74     10319     92.73    150.00      9.26     44.62    500.00     11.80     11.42
+07/25 19:00      0.00      0.00    617.64   2565408    509.79        74     10569     91.96    137.50      6.66     51.95    285.00     11.79     11.42
+07/25 20:00      0.00      0.00    617.62   2563871    509.79        50     10569     89.65    130.40      6.07     58.54     86.00     11.78     11.42
+07/25 21:00      0.00      0.00    617.61   2563103    509.81        25     10319     87.03    128.80      6.81     60.92      1.00     11.79     11.42
+07/25 22:00      0.00      0.00    617.64   2565408    508.62         0     10319     85.80    131.70      6.84     62.32      0.00     11.79     11.42
+07/25 23:00      0.00      0.00    617.61   2563103    502.61         0      1019     84.70    138.90      5.71     62.26      0.00     11.79     11.42
+
+
+
+ + + + Dissemination of this information without the expressed written consent of + the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is + strictly prohibited. +
+ These data are gathered electronically and, as presented, have not + undergone any form of quality control. These data are not guaranteed to be + valid or correct. +
+ Go to home page. +
+ Go to station list. +
+ Go to station map. +

+ + Downstream station: +

+ + Previous Day +
+ Next Day +
+ Current Day +

+ Display graphics. +
+ Display DCP transmissions. +

+ 
+    
+    SHEF ID:       MYST2     Pat Mayse Lake
+    STATION TYPE:  RESERVOIR
+    Flood Control Pool:  Bottom =  451.00, Top =  460.50
+    
+                   PRECIP PRECIP(2) ELEVATION   STORAGE    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
+                   INCHES    INCHES      FEET      ACFT       CFS       CFS     DEG-F   DEGREES       MPH   PERCENT      W/M2     VOLTS     VOLTS
+                     GOES      GOES      GOES      GOES  COMPRESQ  COMPRESQ      GOES      GOES      GOES      GOES      GOES      GOES      GOES
+       (Cen)
+    02/18 01:00      0.00      0.00    447.00     96402         9         0     55.20    173.00     18.02     56.37      0.00     12.99     12.83
+    02/18 02:00      0.00      0.00    446.99     96352         4         0     55.74    169.60     17.20     60.82      0.00     12.96     12.77
+    02/18 03:00      0.00      0.00    447.03     96556         0         0     55.29    170.20     17.79     64.79      0.00     12.94     12.77
+    02/18 04:00      0.00      0.00    447.00     96402         0         0     55.08    166.60     21.41     68.76      0.00     12.93     12.71
+    02/18 05:00      0.00      0.00    446.99     96352         0         0     55.96    166.00     19.19     72.00      0.00     12.92     12.71
+    02/18 06:00      0.00      0.00    446.99     96352         0         0     55.08    159.30     14.64     74.56      0.00     12.92     12.71
+    02/18 07:00      0.00      0.00    446.99     96352         0         0     55.40    154.30     13.20     77.19      0.00     12.91     12.71
+    02/18 08:00      0.00      0.00    446.99     96352        61         0     55.96    157.70     12.84     78.65     10.00     12.92     12.71
+    02/18 09:00      0.00      0.00    447.00     96402        61         0     57.96    171.90     16.03     78.05     32.00     13.10     12.71
+    02/18 10:00      0.00      0.00    447.00     96402       185         0     58.71    180.90     18.72     78.66     57.00     13.46     12.89
+    02/18 11:00      0.00      0.00    447.02     96505       188         0     62.56    198.90     23.01     73.84     79.00     13.68     13.24
+    02/18 12:00      0.00      0.00    447.03     96556       188         0     63.77    197.60     24.16     71.14    428.00     14.17     13.48
+    02/18 13:00      0.00      0.00    447.07     96762       128         0     66.29    214.80     29.18     58.32    662.00     13.73     13.94
+    02/18 14:00      0.00      0.00    447.07     96762       128         0     68.27    225.70     26.96     52.27    489.00     13.69     13.53
+    02/18 15:00      0.00      0.00    447.10     96917         3         0     69.69    273.90     23.58     31.94    575.00     13.76     13.48
+    02/18 16:00      0.00      0.00    447.01     96454         0         0     68.94    286.00     19.61     29.62    462.00     13.83     13.53
+    02/18 17:00      0.00      0.00    447.01     96454         0         0     63.77    312.20     19.79     32.12    264.00     13.86     13.59
+    02/18 18:00      0.00      0.00    446.98     96302         0         0     59.70    321.70     16.90     37.19     47.00     13.71     13.65
+    02/18 19:00      0.00      0.00    446.99     96352         0         0     56.17    343.30     11.96     42.25      0.00     13.26     13.53
+    02/18 20:00      0.00      0.00    446.98     96302         0         0     53.44    343.30     10.10     46.47      0.00     13.12     13.07
+    02/18 21:00      0.00      0.00    446.99     96352         0         0     50.90    338.00      6.26     50.50      0.00     13.08     12.89
+    02/18 22:00      0.00      0.00    446.98     96302         0         0     49.59    335.90      6.32     51.17      0.00     13.05     12.89
+    02/18 23:00      0.00      0.00    446.98     96302         0         0     47.50    342.30      2.99     52.46      0.00     13.03     12.89
+    02/19 00:00      0.00      0.00    446.98     96302         0         0     45.95    347.50      6.09     53.49      0.00     13.01     12.65
+    
+
+
+
+ + diff --git a/test/files/usace/swtwc/DSNT2.current.html b/test/files/usace/swtwc/DSNT2.current.html new file mode 100644 index 00000000..05710bca --- /dev/null +++ b/test/files/usace/swtwc/DSNT2.current.html @@ -0,0 +1,84 @@ + + + + + DSNT2: Lake Texoma, Denison Dam (tabular) + + + +

DSNT2: Lake Texoma, Denison Dam

+

Data processed at Fri Aug 25 2017 16:12

+
+ All information presented herein is the property of the government of the + United States of America and may be revised or deleted at any time without + prior notice. +
+ Dissemination of this information without the expressed written consent of + the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is + strictly prohibited. +
+ These data are gathered electronically and, as presented, have not + undergone any form of quality control. These data are not guaranteed to be + valid or correct. +
+ Go to home page. +
+ Go to station list. +

+ Upstream stations: +

+ + Downstream station: + + +
+ Previous Day +
+ Next Day +
+ Current Day +
+ 
+    
+SHEF ID:       DSNT2     Lake Texoma, Denison Dam
+STATION TYPE:  Reservoir
+Normal Pool:  Bottom =   590.00, Top =   616.85
+
+               PRECIP PRECIP(A) ELEVATION   STORAGE   TW-ELEV    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
+                   in        in        ft     ac-ft        ft       cfs       cfs         F       deg       mph         %      W/m2      volt      volt
+              Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev  Regi-Rev  Regi-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev   Ccp-Rev
+(US/Central)
+08/25 00:00      0.00      0.00    617.90   2585386    508.67      5230     10069     75.60    109.60      6.01     82.04      0.00     13.73     13.18
+08/25 01:00      0.00      0.00    617.89   2584618    502.79      -901      -901     74.61    111.90      3.24     85.22      1.00     13.72     13.18
+08/25 02:00      0.00      0.00    617.89   2584618    502.00      -901      -901     73.27    108.20      5.65     86.56      0.00     13.72     13.18
+08/25 03:00      0.00      0.00    617.88   2583849    501.97      -901      -901     72.07     93.10      2.83     89.55      0.00     13.72     13.18
+08/25 04:00      0.00      0.00    617.88   2583849    501.97      -901      -901     71.96    110.90      4.25     87.29      0.00     13.72     13.18
+08/25 05:00      0.00      0.00    617.88   2583849    501.97      -901      -901     72.18    109.60      6.12     81.86      0.00     13.72     13.18
+08/25 06:00      0.00      0.00    617.86   2582313    504.50      -901      -901     71.96    107.90      6.39     77.95      1.00     13.72     13.18
+08/25 07:00      0.00      0.00    617.89   2584618    509.66      -901      -901     70.86    107.80      4.77     79.29      2.00     13.72     13.18
+08/25 08:00      0.00      0.00    617.88   2583849    509.88      -901      -901     70.65    109.60      6.14     77.83     64.00     13.72     13.18
+08/25 09:00      0.00      0.00    617.87   2583081    509.86      -901      -901     71.85    112.20      6.36     73.98    108.00     13.72     13.18
+08/25 10:00      0.00      0.00    617.85   2581544    509.88      -901      -901     74.39    108.80      8.57     67.27    320.00     13.72     13.18
+08/25 11:00      0.00      0.00    617.86   2582313    509.88      -901      -901     77.45    103.00      5.56     62.08    566.00     13.72     13.18
+08/25 12:00      0.00      0.00    617.85   2581544    509.87      -901      -901     79.21    112.40      6.97     60.55    524.00     13.72     13.18
+08/25 13:00      0.00      0.00    617.86   2582313    509.88      -901      -901     81.97    122.80      6.66     58.97   1128.00     13.72     13.18
+08/25 14:00      0.00      0.00    617.84   2580776    509.87      -901      -901     81.86    106.40      6.95     57.14    886.00     13.72     13.18
+08/25 15:00      0.00      0.00    617.84   2580776    509.90      -901      -901     83.39    123.70      7.21     56.65    723.00     13.72     13.18
+08/25 16:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 17:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 18:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 19:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 20:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 21:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 22:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+08/25 23:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
+
+
+
+ + diff --git a/test/files/usace/swtwc/MYST2.20130218.html b/test/files/usace/swtwc/MYST2.20130218.html deleted file mode 100644 index d2cc696b..00000000 --- a/test/files/usace/swtwc/MYST2.20130218.html +++ /dev/null @@ -1,84 +0,0 @@ - - - - MYST2: Pat Mayse Lake (tabular) - - - -

MYST2: Pat Mayse Lake

-

Data processed at Tue Feb 19 15:07:26 2013 Central Standard Time

-
- All information presented herein is the property of the government of the - United States of America and may be revised or deleted at any time without - prior notice. -
- Dissemination of this information without the expressed written consent of - the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is - strictly prohibited. -
- These data are gathered electronically and, as presented, have not - undergone any form of quality control. These data are not guaranteed to be - valid or correct. -
- Go to home page. -
- Go to station list. -
- Go to station map. -

- - Downstream station: -

- - Previous Day -
- Next Day -
- Current Day -

- Display graphics. -
- Display DCP transmissions. -

- 
-    
-    SHEF ID:       MYST2     Pat Mayse Lake
-    STATION TYPE:  RESERVOIR
-    Flood Control Pool:  Bottom =  451.00, Top =  460.50
-    
-                   PRECIP PRECIP(2) ELEVATION   STORAGE    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
-                   INCHES    INCHES      FEET      ACFT       CFS       CFS     DEG-F   DEGREES       MPH   PERCENT      W/M2     VOLTS     VOLTS
-                     GOES      GOES      GOES      GOES  COMPRESQ  COMPRESQ      GOES      GOES      GOES      GOES      GOES      GOES      GOES
-       (Cen)
-    02/18 01:00      0.00      0.00    447.00     96402         9         0     55.20    173.00     18.02     56.37      0.00     12.99     12.83
-    02/18 02:00      0.00      0.00    446.99     96352         4         0     55.74    169.60     17.20     60.82      0.00     12.96     12.77
-    02/18 03:00      0.00      0.00    447.03     96556         0         0     55.29    170.20     17.79     64.79      0.00     12.94     12.77
-    02/18 04:00      0.00      0.00    447.00     96402         0         0     55.08    166.60     21.41     68.76      0.00     12.93     12.71
-    02/18 05:00      0.00      0.00    446.99     96352         0         0     55.96    166.00     19.19     72.00      0.00     12.92     12.71
-    02/18 06:00      0.00      0.00    446.99     96352         0         0     55.08    159.30     14.64     74.56      0.00     12.92     12.71
-    02/18 07:00      0.00      0.00    446.99     96352         0         0     55.40    154.30     13.20     77.19      0.00     12.91     12.71
-    02/18 08:00      0.00      0.00    446.99     96352        61         0     55.96    157.70     12.84     78.65     10.00     12.92     12.71
-    02/18 09:00      0.00      0.00    447.00     96402        61         0     57.96    171.90     16.03     78.05     32.00     13.10     12.71
-    02/18 10:00      0.00      0.00    447.00     96402       185         0     58.71    180.90     18.72     78.66     57.00     13.46     12.89
-    02/18 11:00      0.00      0.00    447.02     96505       188         0     62.56    198.90     23.01     73.84     79.00     13.68     13.24
-    02/18 12:00      0.00      0.00    447.03     96556       188         0     63.77    197.60     24.16     71.14    428.00     14.17     13.48
-    02/18 13:00      0.00      0.00    447.07     96762       128         0     66.29    214.80     29.18     58.32    662.00     13.73     13.94
-    02/18 14:00      0.00      0.00    447.07     96762       128         0     68.27    225.70     26.96     52.27    489.00     13.69     13.53
-    02/18 15:00      0.00      0.00    447.10     96917         3         0     69.69    273.90     23.58     31.94    575.00     13.76     13.48
-    02/18 16:00      0.00      0.00    447.01     96454         0         0     68.94    286.00     19.61     29.62    462.00     13.83     13.53
-    02/18 17:00      0.00      0.00    447.01     96454         0         0     63.77    312.20     19.79     32.12    264.00     13.86     13.59
-    02/18 18:00      0.00      0.00    446.98     96302         0         0     59.70    321.70     16.90     37.19     47.00     13.71     13.65
-    02/18 19:00      0.00      0.00    446.99     96352         0         0     56.17    343.30     11.96     42.25      0.00     13.26     13.53
-    02/18 20:00      0.00      0.00    446.98     96302         0         0     53.44    343.30     10.10     46.47      0.00     13.12     13.07
-    02/18 21:00      0.00      0.00    446.99     96352         0         0     50.90    338.00      6.26     50.50      0.00     13.08     12.89
-    02/18 22:00      0.00      0.00    446.98     96302         0         0     49.59    335.90      6.32     51.17      0.00     13.05     12.89
-    02/18 23:00      0.00      0.00    446.98     96302         0         0     47.50    342.30      2.99     52.46      0.00     13.03     12.89
-    02/19 00:00      0.00      0.00    446.98     96302         0         0     45.95    347.50      6.09     53.49      0.00     13.01     12.65
-    
-
-
-
- - diff --git a/test/files/usace/swtwc/MYST2.current.html b/test/files/usace/swtwc/MYST2.current.html deleted file mode 100644 index f79ecfca..00000000 --- a/test/files/usace/swtwc/MYST2.current.html +++ /dev/null @@ -1,84 +0,0 @@ - - - - MYST2: Pat Mayse Lake (tabular) - - - -

MYST2: Pat Mayse Lake

-

Data processed at Mon Mar 04 15:07:26 2013 Central Standard Time

-
- All information presented herein is the property of the government of the - United States of America and may be revised or deleted at any time without - prior notice. -
- Dissemination of this information without the expressed written consent of - the Public Affairs Office, Tulsa District U.S. Army Corps of Engineers is - strictly prohibited. -
- These data are gathered electronically and, as presented, have not - undergone any form of quality control. These data are not guaranteed to be - valid or correct. -
- Go to home page. -
- Go to station list. -
- Go to station map. -

- - Downstream station: -

- - Previous Day -
- Next Day -
- Current Day -

- Display graphics. -
- Display DCP transmissions. -

- 
-    
-    SHEF ID:       MYST2     Pat Mayse Lake
-    STATION TYPE:  RESERVOIR
-    Flood Control Pool:  Bottom =  451.00, Top =  460.50
-    
-                   PRECIP PRECIP(2) ELEVATION   STORAGE    INFLOW   RELEASE  AIR-TEMP  WIND-DIRWIND-SPEED REL-HUMID SOLAR-RAD   VOLTAGE  BAT-LOAD
-                   INCHES    INCHES      FEET      ACFT       CFS       CFS     DEG-F   DEGREES       MPH   PERCENT      W/M2     VOLTS     VOLTS
-                     GOES      GOES      GOES      GOES  COMPRESQ  COMPRESQ      GOES      GOES      GOES      GOES      GOES      GOES      GOES
-       (Cen)
-    03/04 01:00      0.00      0.00    446.84     95602         0         0     56.84    174.90     17.27     49.83      0.00     12.96     12.77
-    03/04 02:00      0.00      0.00    446.85     95652         0         0     56.28    172.70     18.93     51.60      0.00     12.93     12.71
-    03/04 03:00      0.00      0.00    446.85     95652         0         0     55.08    172.70     13.84     55.26      0.00     12.91     12.71
-    03/04 04:00      0.00      0.00    446.85     95652         0         0     54.86    177.40     12.90     60.88      0.00     12.90     12.71
-    03/04 05:00      0.00      0.00    446.85     95652         0         0     55.53    188.60     17.42     63.32      0.00     12.89     12.65
-    03/04 06:00      0.00      0.00    446.85     95652         0         0     55.40    188.60     22.32     66.99      0.00     12.88     12.65
-    03/04 07:00      0.00      0.00    446.84     95602         0         0     55.18    190.00     21.77     69.92      3.00     12.89     12.65
-    03/04 08:00      0.00      0.00    446.84     95602         0         0     55.74    189.80     21.15     71.63     44.00     13.13     12.65
-    03/04 09:00      0.00      0.00    446.86     95702         0         0     56.62    191.10     20.28     72.61     80.00     13.72     12.89
-    03/04 10:00      0.00      0.00    446.84     95602         0         0     58.37    195.70     21.02     69.19    170.00     14.12     13.53
-    03/04 11:00      0.00      0.00    446.85     95652         0         0     60.57    203.30     18.10     61.43    233.00     14.05     13.88
-    03/04 12:00      0.00      0.00    446.86     95702         0         0     64.44    209.00     20.82     51.36    747.00     13.69     13.83
-    03/04 13:00      0.00      0.00    446.89     95852         0         0     68.38    221.20     24.86     38.78    846.00     13.70     13.71
-    03/04 14:00      0.00      0.00    446.84     95602         0         0     70.03    212.20     23.34     36.33    744.00     13.70     13.83
-    03/04 15:00      0.00      0.00    446.87     95752      ----      ----     72.45    215.40     19.07     37.01    704.00     13.68     13.77
-    03/04 16:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 17:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 18:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 19:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 20:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 21:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 22:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/04 23:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    03/05 00:00      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----      ----
-    
-
-
-
- - diff --git a/test/usace_swtwc_test.py b/test/usace_swtwc_test.py index be2ee6a2..e79cb4aa 100644 --- a/test/usace_swtwc_test.py +++ b/test/usace_swtwc_test.py @@ -21,26 +21,27 @@ def test_get_stations(): def test_get_station_data(): test_station_data = [ - ('MYST2', '2013-02-18', { - 'code': 'MYST2', - 'description': 'Pat Mayse Lake', - 'station_type': 'RESERVOIR', - 'timezone': 'Cen', + ('DSNT2', '2017-07-25', { + 'code': 'DSNT2', + 'description': 'Lake Texoma, Denison Dam', + 'station_type': 'Reservoir', + 'timezone': 'US/Central', 'values': { - '2013-02-18 01:00:00': { - 'AIR-TEMP': 55.2, - 'BAT-LOAD': 12.83, - 'ELEVATION': 447.0, - 'INFLOW': 9.0, + '2017-07-25 01:00:00': { + 'AIR-TEMP': 80.10, + 'BAT-LOAD': 11.37, + 'ELEVATION': 617.73, + 'INFLOW': 564, 'PRECIP': 0.0, - 'PRECIP(2)': 0.0, - 'REL-HUMID': 56.37, - 'RELEASE': 0.0, + 'PRECIP(A)': 0.0, + 'REL-HUMID': 81.98, + 'RELEASE': 19, 'SOLAR-RAD': 0.0, - 'STORAGE': 96402.0, - 'VOLTAGE': 12.99, - 'WIND-DIR': 173.0, - 'WIND-SPEED': 18.02 + 'STORAGE': 2572324.0, + 'VOLTAGE': 11.77, + 'WIND-DIR': 135.90, + 'WIND-SPEED': 1.42, + 'TW-ELEV': 501.87 }, }, }), @@ -64,9 +65,9 @@ def test_get_station_data_current(): # can't easily test current since it is a moving target changes, but mostly # just make sure it parses correctl: current will have '---' values where # previous days do not - data_file = 'usace/swtwc/MYST2.current.html' + data_file = 'usace/swtwc/DSNT2.current.html' with test_util.mocked_urls(data_file): - station_data = ulmo.usace.swtwc.get_station_data('MYST2') + station_data = ulmo.usace.swtwc.get_station_data('DSNT2') assert len(station_data.get('values')) > 0 diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index cce1e1d4..63194f9e 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -68,6 +68,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): filename = '%s.%s.html'% (station_code, date_str) data_url = 'http://www.swt-wc.usace.army.mil/webdata/gagedata/' + filename + print data_url path = os.path.join(USACE_SWTWC_DIR, filename) with util.open_file_for_url(data_url, path) as f: @@ -90,6 +91,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): second_line = sio.readline() station_dict['station_type'] = second_line.strip().split(':')[1].strip() + print station_dict notes = [] @@ -103,9 +105,9 @@ def get_station_data(station_code, date=None, as_dataframe=False): if len(notes): station_dict['notes'] = '\n'.join(notes) - variable_names = _split_line(sio.readline()[15:], 10) - variable_units = _split_line(sio.readline()[15:], 10) - variable_sources = _split_line(sio.readline()[15:], 10) + variable_names = _split_line(sio.readline()[11:], 10) + variable_units = _split_line(sio.readline()[11:], 10) + variable_sources = _split_line(sio.readline()[11:], 10) station_dict['variables'] = dict([ (name, {'unit': unit, 'source': source}) @@ -115,7 +117,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): station_dict['timezone'] = sio.readline().strip().strip('()') column_names = ['datetime'] + variable_names - widths = [15] + ([10] * len(variable_names)) + widths = [14] + ([10] * len(variable_names)) converters = dict([ (variable_name, lambda x: float(x) if x != '----' else np.nan) for variable_name in variable_names From b5b5902a2f820eba1eb3efbd2a6a98deee7c4afc Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Fri, 25 Aug 2017 18:02:30 -0500 Subject: [PATCH 4/8] remove debugging print stmt --- ulmo/usace/swtwc/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index 63194f9e..4c6e7cd4 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -68,7 +68,6 @@ def get_station_data(station_code, date=None, as_dataframe=False): filename = '%s.%s.html'% (station_code, date_str) data_url = 'http://www.swt-wc.usace.army.mil/webdata/gagedata/' + filename - print data_url path = os.path.join(USACE_SWTWC_DIR, filename) with util.open_file_for_url(data_url, path) as f: From d2a7820ccde036a4e2ac0844eb51a21224fd2da6 Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Fri, 25 Aug 2017 18:11:40 -0500 Subject: [PATCH 5/8] remove another debugging print stmt --- ulmo/usace/swtwc/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index 4c6e7cd4..17edb95b 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -90,7 +90,6 @@ def get_station_data(station_code, date=None, as_dataframe=False): second_line = sio.readline() station_dict['station_type'] = second_line.strip().split(':')[1].strip() - print station_dict notes = [] From 41267c7e7cae4b6b59699be737719108ce92be38 Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Thu, 21 Mar 2019 19:13:48 -0500 Subject: [PATCH 6/8] string formatting update --- ulmo/usace/swtwc/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index 17edb95b..b7510f2e 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -66,7 +66,7 @@ def get_station_data(station_code, date=None, as_dataframe=False): date_str = date.strftime('%Y%m%d') year = date.year - filename = '%s.%s.html'% (station_code, date_str) + filename = '{}.{}.html'.format(station_code, date_str) data_url = 'http://www.swt-wc.usace.army.mil/webdata/gagedata/' + filename path = os.path.join(USACE_SWTWC_DIR, filename) From 4589a1abc91d7087087b622e9788c13ec1b396a1 Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Thu, 21 Mar 2019 19:14:18 -0500 Subject: [PATCH 7/8] adding User-Agent header for USACE requests to work --- ulmo/usace/swtwc/core.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index b7510f2e..668ce8fd 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -19,6 +19,7 @@ from bs4 import BeautifulSoup import numpy as np +import requests import pandas from ulmo import util @@ -68,19 +69,22 @@ def get_station_data(station_code, date=None, as_dataframe=False): filename = '{}.{}.html'.format(station_code, date_str) data_url = 'http://www.swt-wc.usace.army.mil/webdata/gagedata/' + filename - path = os.path.join(USACE_SWTWC_DIR, filename) - with util.open_file_for_url(data_url, path) as f: - soup = BeautifulSoup(f) - pre = soup.find('pre') - if pre is None: - error_msg = 'no data could be found for station code %(station_code)s and date %(date)s (url: %(data_url)s)' % { - 'date': date, - 'data_url': data_url, - 'station_code': station_code, - } - raise ValueError(error_msg) - sio = StringIO.StringIO(str(pre.text.strip())) + # requests without User-Agent header get rejected + headers = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36' + } + resp = requests.get(data_url, headers=headers) + soup = BeautifulSoup(resp.content) + pre = soup.find('pre') + if pre is None: + error_msg = 'no data could be found for station code %(station_code)s and date %(date)s (url: %(data_url)s)' % { + 'date': date, + 'data_url': data_url, + 'station_code': station_code, + } + raise ValueError(error_msg) + sio = StringIO.StringIO(str(pre.text.strip())) first_line = sio.readline() split = first_line[8:].strip().split() From a74ecae466322afc158832826d43ce8e99f30c3d Mon Sep 17 00:00:00 2001 From: solomon-negusse Date: Fri, 22 Mar 2019 19:15:11 -0500 Subject: [PATCH 8/8] stations fix --- ulmo/usace/swtwc/core.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ulmo/usace/swtwc/core.py b/ulmo/usace/swtwc/core.py index 668ce8fd..c3ce8b59 100644 --- a/ulmo/usace/swtwc/core.py +++ b/ulmo/usace/swtwc/core.py @@ -150,15 +150,17 @@ def get_stations(): a python dict with station codes mapped to station information """ stations_url = 'http://www.swt-wc.usace.army.mil/shefids.htm' - path = os.path.join(USACE_SWTWC_DIR, 'shefids.htm') - - with util.open_file_for_url(stations_url, path) as f: - soup = BeautifulSoup(f) - pre = soup.find('pre') - links = pre.find_all('a') - stations = [ - _parse_station_link(link) for link in links - ] + + headers = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36' + } + resp = requests.get(stations_url, headers=headers) + soup = BeautifulSoup(resp.content) + pre = soup.find('pre') + links = pre.find_all('a') + stations = [ + _parse_station_link(link) for link in links + ] return dict([ (station['code'], station)