From c76c548e1b9fec767a52b0940ce1cf2be9fd370d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 2 Feb 2023 12:17:09 -0700 Subject: [PATCH 1/9] Per #2426, port bugfix over for the develop branch. --- data/table_files/ndbc_stations.xml | 12 +++ src/tools/other/ascii2nc/ndbc_handler.cc | 103 ++++++++++++----------- 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/data/table_files/ndbc_stations.xml b/data/table_files/ndbc_stations.xml index 7e73ef73a3..c536e399f4 100644 --- a/data/table_files/ndbc_stations.xml +++ b/data/table_files/ndbc_stations.xml @@ -260,6 +260,7 @@ + @@ -475,6 +476,7 @@ + @@ -777,6 +779,7 @@ + @@ -789,6 +792,7 @@ + @@ -919,9 +923,11 @@ + + @@ -946,6 +952,7 @@ + @@ -1113,11 +1120,13 @@ + + @@ -1231,6 +1240,7 @@ + @@ -1333,6 +1343,7 @@ + @@ -1341,6 +1352,7 @@ + diff --git a/src/tools/other/ascii2nc/ndbc_handler.cc b/src/tools/other/ascii2nc/ndbc_handler.cc index f76df3c4d8..cd9f960f90 100644 --- a/src/tools/other/ascii2nc/ndbc_handler.cc +++ b/src/tools/other/ascii2nc/ndbc_handler.cc @@ -87,11 +87,11 @@ NdbcHandler::NdbcHandler(const string &program_name) : // read in and parse the locations file if (!locations.initialize(locationsFileName)) { - mlog << Error << "\ncould not initialize station loations file\n\n"; + mlog << Error << "\nCannot initialize NDBC station loations file: " + << locationsFileName << "\n\n"; exit(1); } - //locations.print(); - + // // store column info for all the data columns (column names) // NOTE these will be used as index values in the observations @@ -215,13 +215,14 @@ bool NdbcHandler::_readObservations(LineDataFile &ascii_file) //////////////////////////////////////////////////////////////////////// bool NdbcHandler::_parseObservationLineStandard(DataLine &data_line, - const string &filename) + const string &filename) { string method_name = "NdbcHandler::_parseObservationLineStandard() "; if (format_version != NDBC_FORMAT_VERSION_STANDARD) { - mlog << Error << "\n" << method_name << "->" - << "Standard NDBC format is the only supported format\n\n"; + mlog << Warning << "\n" << method_name << "->" + << "Standard NDBC format is the only supported format: " + << filename << "\n\n"; return false; } @@ -229,11 +230,11 @@ bool NdbcHandler::_parseObservationLineStandard(DataLine &data_line, // Make sure that the line contains the correct number of tokens // if (data_line.n_items() != NUM_COLS_STANDARD) { - mlog << Error << "\n" << method_name << "-> " - << "line number " << data_line.line_number() - << " does not have the correct number of columns " << data_line.n_items() - << " (" << NUM_COLS_STANDARD << "). Skipping this line in \"" - << filename << "\".\n\n"; + mlog << Warning << "\n" << method_name << "-> " + << "Skipping line number " << data_line.line_number() + << " with an unexpected number of columns (" + << data_line.n_items() << " != " << NUM_COLS_STANDARD << "): " + << filename << "\n\n"; return false; } @@ -242,10 +243,10 @@ bool NdbcHandler::_parseObservationLineStandard(DataLine &data_line, // time_t valid_time = _getValidTime(data_line); if (valid_time == 0) { - mlog << Error << "\n" << method_name << "-> " - << "line number " << data_line.line_number() - << " time could not be parsed, skipping this line in \"" - << filename << "\".\n\n"; + mlog << Warning << "\n" << method_name << "-> " + << "Skipping line number " << data_line.line_number() + << " whose vaild time cannot not be parsed: " + << filename << "\n\n"; return false; } @@ -263,9 +264,9 @@ bool NdbcHandler::_parseObservationLineStandard(DataLine &data_line, name = column[i].name; grib_code = i; // it's not actually grib code, its obs_vid, according to howard _addObservations(Observation(header_type, stationId, valid_time, - stationLat, stationLon, stationAlt, - quality_flag, grib_code, pressure_level_hpa, - height_m, value, name)); + stationLat, stationLon, stationAlt, + quality_flag, grib_code, pressure_level_hpa, + height_m, value, name)); } return true; } @@ -287,15 +288,15 @@ bool NdbcHandler::_setStationInfo(const string &filename) // expect .txt as the name i0 = fname.find(".txt"); if (i0 == string::npos) { - mlog << Error << "\n" << "expect file name of format '.txt'\n" - << "Got " << fname << "\n\n"; + mlog << Warning << "\n" << "NDBC file name does not follow the " + << "expected '.txt' format: " << fname << "\n\n"; return false; } stationId = fname.substr(0, i0); if (!locations.lookupLatLonElev(stationId, stationLat, stationLon, - stationAlt)) { - mlog << Error << "\n" << "No location information found for station " - << stationId << " do not process file " << filename << "\n\n"; + stationAlt)) { + mlog << Warning << "\n" << "NDBC station " << stationId + << " location information not found: " << filename << "\n\n"; return false; } return true; @@ -323,8 +324,8 @@ bool NdbcHandler::_determineFileType(LineDataFile &ascii_file) } } format_version = NDBC_FORMAT_VERSION_UNKNOWN; - mlog << Error << "\nNdbcHandler::_determineFileType -> " - << "Unknown file type\n\n"; + mlog << Warning << "\nNdbcHandler::_determineFileType -> " + << "Unknown file type: " << ascii_file.filename() << "\n\n"; return false; } @@ -335,10 +336,10 @@ time_t NdbcHandler::_getValidTime(const DataLine &data_line) const // // Pull out the date information // - if (column_pointer_year < 0 || column_pointer_month < 0 || column_pointer_day < 0 || + if (column_pointer_year < 0 || column_pointer_month < 0 || column_pointer_day < 0 || column_pointer_hour < 0 || column_pointer_minute < 0) { - mlog << Error << "\nNdbcHandler::_getValidTime -> " - << "Not all time related column pointers are set\n\n"; + mlog << Warning << "\nNdbcHandler::_getValidTime -> " + << "Not all time related column pointers are set.\n\n"; return 0; } string year = _extractColumn(data_line, column_pointer_year); @@ -391,9 +392,9 @@ bool NdbcHandler::_readHeaderInfo(LineDataFile &ascii_file) // The first line of the file contains the headers // if (!(ascii_file >> data_line)) { - mlog << Error << "\nNdbcHandler::_readHeaderInfo() -> " - << "error reading header line from input ASCII file \"" - << ascii_file.filename() << "\"\n\n"; + mlog << Warning << "\nNdbcHandler::_readHeaderInfo() -> " + << "Problem reading header line from input ASCII file: " + << ascii_file.filename() << "\n\n"; return false; } @@ -401,9 +402,10 @@ bool NdbcHandler::_readHeaderInfo(LineDataFile &ascii_file) // Check for the correct number of columns in the header line // if (data_line.n_items() != NUM_COLS_STANDARD) { - mlog << Error << "\nNdbcHandler::_readHeaderInfo() -> " - << "NDBC file has incorrect number of columns (" - << data_line.n_items() << ") in header line\n\n"; + mlog << Warning << "\nNdbcHandler::_readHeaderInfo() -> " + << "Unexpected number of header columns (" << data_line.n_items() + << " != " << NUM_COLS_STANDARD << "): " + << ascii_file.filename() << "\n\n"; return false; } @@ -426,30 +428,33 @@ bool NdbcHandler::_readHeaderInfo(LineDataFile &ascii_file) } else { bool found = false; for (size_t j=0; j " - << "NDBC file has unknown header item " << s << "\n\n"; - status = false; + mlog << Warning << "\nNdbcHandler::_readHeaderInfo() -> " + << "Unexpected header column (" << s << "): " + << ascii_file.filename() << "\n\n"; + status = false; } } } - if (column_pointer_year == -1 || column_pointer_month == -1 || - column_pointer_day == -1 || column_pointer_hour == -1 || + if (column_pointer_year == -1 || column_pointer_month == -1 || + column_pointer_day == -1 || column_pointer_hour == -1 || column_pointer_minute == -1) { - mlog << Error << "\nNdbcHandler::_readHeaderInfo() -> " - << "NDBC file did not have all time fields in header \n\n"; - status = false; + mlog << Warning << "\nNdbcHandler::_readHeaderInfo() -> " + << "NDBC file did not have all time fields in header: " + << ascii_file.filename() << "\n\n"; + status = false; } for (size_t j=0; j " - << "NDBC file did not have all expected fields in header \n\n"; + mlog << Warning << "\nNdbcHandler::_readHeaderInfo() -> " + << "NDBC file did not have all expected fields in header: " + << ascii_file.filename() << "\n\n"; status = false; break; } From 8bca0e90d3b55545a2cdd585ccd953589c3898e4 Mon Sep 17 00:00:00 2001 From: David Albo Date: Fri, 17 Feb 2023 16:43:26 -0700 Subject: [PATCH 2/9] initial commit of a script designed to pull NDBC station info from the web and update local data --- .../utility/build_ndbc_stations_from_web.py | 491 ++++++++++++++++++ 1 file changed, 491 insertions(+) create mode 100755 scripts/utility/build_ndbc_stations_from_web.py diff --git a/scripts/utility/build_ndbc_stations_from_web.py b/scripts/utility/build_ndbc_stations_from_web.py new file mode 100755 index 0000000000..19f70b5827 --- /dev/null +++ b/scripts/utility/build_ndbc_stations_from_web.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 + +''' +Created on February 17, 2023 + +@author: davealbo + +The script reads NDBC station information from two NOAA websites and merges the contents into one local list. +The list contains latitude, longitude and elevation data for all known stations. +The local list can be read by ascii2nc for processing of NDBC data. +Algorithm: + Pull down active station xml file from web and create a list of active information. + Optionally write the list to an active stations text file. + + Pull down complete index list from web. + for each file refered to in the complete index list contents: + pull down that stations web page data and append to list of complete station information + optionally write the list of complete station info to a text file. + optionally save all the individual web page data that was pulled down into a subdirectory. + + warn about stations that are both active and complete but have different values + create a master list of all the unique stations from both lists, defer to active for any conflicts + write the master list +''' + +from optparse import OptionParser +import os +import shutil +import shlex +import errno +from subprocess import Popen, PIPE + +# this needs to change! +# hardwired location of current default stations file +DEFAULT_STATIONS_FILE = "../../data/table_files/ndbc_stations.xml" + +# hardwired NOAA top webpage +TOP_WEBSITE = "https://www.ndbc.noaa.gov" + +# hardwired website with active station xml +ACTIVE_WEBSITE = "https://www.ndbc.noaa.gov/activestations.xml" + +# hardwired result of a wget of ACTIVE_WEBSITE +ACTIVE_STATIONS_XML = "activestations.xml" + +# hardwired website with index to a complete list of stations +COMPLETE_INDEX_WEBSITE = "https://www.ndbc.noaa.gov/to_station.shtml" + +# hardwired result of a wget of COMPLETE_INDEX_WEBSITE +COMPLETE_STATIONS_INDEX_INFO = "to_station.shtml" + +# hardwired name of optionally saved active stations +ACTIVE_TEXT_FILE = "active.txt" + +# hardwired name of optionally saved complete stations +COMPLETE_TEXT_FILE = "complete.txt" + +# default output file name +DEFAULT_OUTPUT_FILE = "merged.txt" + +MISSING = -99.9 + +def usage(): + print(f'Usage: BuildNdbcStationsFromWeb.py <--save_complete_stations> <--save_intermediate_lists> <--out=out_filename>') + print(f' -s/--save_intermediate_lists: save intermediate active and complete text files, as well as a subdirectory with all the individual complete stations full information (optional, default: False)"') + print(f' if True files {ACTIVE_TEXT_FILE} and ${COMPLETE_TEXT_FILE} and contents of subdir ./stations are saved, otherwise deleted"') + print(f' -o/--out=out_filename: save final text into the named file (default: file name is {DEFAULT_OUTPUT_FILE})"') + print(f' Note: <> indicates optional arguments') + +#---------------------------------------------- +def create_parser_options(parser): + parser.add_option("-i", "--save_intermediate_lists", dest="save_intermediate_lists", + action="store_true", default=False, + help=" save intermediate files pulled from web:" + ACTIVE_TEXT_FILE + "," + COMPLETE_TEXT_FILE + "and save individual complete station info pulled from the web into a subdirectory ./stations (optional, default: False)") + parser.add_option("-o", "--out", dest="out_file", + default=DEFAULT_OUTPUT_FILE, help=" Save the text into the named file (default: " + DEFAULT_OUTPUT_FILE +" )") + parser.add_option("-H", "--Help", dest="full_usage", action="store_true", default=False, help = " show more usage information (optional, default = False)") + return parser.parse_args() + +#---------------------------------------------- +def doCmd(cmd, debug=False): + + #print(cmd) + my_env = os.environ.copy() + args = shlex.split(cmd) + proc = Popen(args, stdout=PIPE, stderr=PIPE, env=my_env) + out, err = proc.communicate() + exitcode = proc.returncode + if exitcode == 0: + return str(out) + else: + if debug: + print("Command failed ", cmd) + return "" + +#---------------------------------------------- +def main(save_intermediate_lists, out_file): + status = True + + # pull the active stations xml from the web + cmd = "wget " + ACTIVE_WEBSITE + print(cmd) + s = doCmd(cmd, True) + if not s: + status = False + + # pull the complete stations html from the web + cmd = "wget " + COMPLETE_INDEX_WEBSITE + print(cmd) + s = doCmd(cmd, True) + if not s: + status = False + if not status: + print("ERROR reading web content") + os.exit(1) + + # prepare to compare to the default stations file to see what has changed + [ids0, lats0, lons0, elevs0] = parse(DEFAULT_STATIONS_FILE) + + + # parse the active stations XML to create a list + [ids, lats, lons, elevs] = processActive(save_intermediate_lists) + + # read the complete stations html, find all the individual stations web links, + # pull each stations data, parse that downloaded station content to create a list + [ids2, lats2, lons2, elevs2] = processComplete(save_intermediate_lists) + + # see which ids are not in complete from active, and which have different lat/lons + # note the one used if that happens is always the active one + for id in ids: + i1 = ids.index(id) + if id in ids2: + i2 = ids2.index(id) + if (lats[i1] != lats2[i2]) or (lons[i1] != lons2[i2]) or (elevs[i1] != elevs2[i2]): + print("latlonelev disagree for ", id, " active:(", lats[i1], ",", lons[i1], ",", elevs[i1], + ") complete:(", lats2[i2], ",", lons2[i2], ",", elevs2[i2], ")") + + # see which ids are not in active but are in complete, make a list of those as ones to merge + toMergeIndex = [] + for id in ids2: + i2 = ids2.index(id) + if id in ids: + i1 = ids.index(id) + else: + toMergeIndex.append(i2) + + print("Merging ", len(toMergeIndex), " items from complete into active") + + for i in toMergeIndex: + id = ids2[i] + lat = lats2[i] + lon = lons2[i] + elev = elevs2[i] + ids.append(id) + lats.append(lat) + lons.append(lon) + elevs.append(elev) + + #now write out the full meal deal by creating a string list + nout = 0 + txtAll = [] + for id in ids: + i = ids.index(id) + station = id + lat = lats[i] + lon = lons[i] + elev = elevs[i] + txt = ''.format(a=station,b=lat,c=lon,d=elev) + txtAll.append(txt) + nout = nout + 1 + + # sort for ease of use + txtAll.sort() + fout = open(out_file, "w") + for txt in txtAll: + fout.write(txt+"\n") + fout.close() + print("Done, wrote out ", nout, " total items to ", out_file) + + + print("Comparing to current default stations") + for id in ids0: + if id not in ids: + print("Station vanished from web:", id) + else: + i0 = ids0.index(id) + station = id + lat0 = lats0[i0] + lon0 = lons0[i0] + elev0 = elevs0[i0] + i = ids.index(id) + lat = lats[i] + lon = lons[i] + elev = elevs[i] + if (lat0 != lat or lon0 != lon or elev0 != elev): + print("latlonelev disagree for ", id, " web:(", lat, ",", lon, ",", elev, + ") default:(", lat0, ",", lon0, ",", elev0, ")") + for id in ids: + if id not in ids0: + print("New station on web not in defaults:", id) + + return 0 + +#---------------------------------------------------- +def processComplete(save_intermediate): + ''' + read the complete stations html, find all the individual stations web links, + pull each stations data, parse that downloaded station content to create a list + ''' + + # initialize these lists to empty + ids = [] + lats = [] + lons = [] + elevs = [] + + # Open the file with the list of php pages online (or local files pulled down) + with open(COMPLETE_STATIONS_INDEX_INFO, 'r') as file: + data = file.read().replace('\n', '') + file.close() + + # start at the beginning + index = 0 + txtAll = [] + while index < len(data): + # pull down another stations info if you can, and parse it + [index, lat, lon, elev, station] = createNextStationInfo(data, index) + if index == -1: + break + + if station: + # form a string and append that plus all the individual stuff to lists + txt = ''.format(a=station,b=lat,c=lon,d=elev) + txtAll.append(txt) + ids.append(station) + lats.append(lat) + lons.append(lon) + elevs.append(elev) + + if save_intermediate: + # keep the subdirectory of individual stations information + + # sort the list for ease of use, then write it + txtAll.sort() + fout = open(COMPLETE_TEXT_FILE, "w") + for txt in txtAll: + fout.write(txt+"\n") + fout.close() + + + else: + # delete the subdirectory stations + path = os.getcwd() + "/stations" + if (os.path.exists(path)): + try: + shutil.rmtree(path) + except: + print('WARNING: ' + path + ' not completely cleaned out.') + + return [ids, lats, lons, elevs] + +#---------------------------------------------- +def createNextStationInfo(data, i): + + lat = MISSING + lon = MISSING + elev = MISSING + station = "" + + #data has entries like this: 45001 + #on entry i points to the starting location within data to start looking + index = data.find('href="station_page.php?', i) + if index == -1: + return [-1, lat, lon, elev, station] + + # the stuff beyond 'href="' is the file name that you get via wget, followed by another '"' + index2 = index + 6 # point to 'station_page' + index3 = data.find('">', index2) # point to " at end (which is followed by >) + + index = index3 + 3 # set index for return to beyond this + + # what to go for online: + ref = TOP_WEBSITE + '/' + data[index2:index3] + + # name of returned file + filename = data[index2:index3] + + # go get it + cmd = 'wget "' + ref + '"' + print(cmd) + s = doCmd(cmd, True) + if not s: + # note try to keep going forward as index has been updated + print("ERROR data not online: ", ref) + return [index, lat, lon, elev, station] + + # make a stations subdirectory if needed + cwd = os.getcwd() + if not makeDirIfNeeded(cwd + "/stations"): + print("ERROR cannot create needed subdirectory 'stations'") + return [index, lat, lon, elev, station] + + # move this file to the stations subdirectory + shutil.move(cwd + "/" + filename, cwd + "/stations/" + filename) + + # parse the file and return the information, including the next index + return parseStationInfo(filename, index) + +#---------------------------------------------------------------------------- +def makeDirIfNeeded(path, debug=False): + if (debug): + print("Making directory if needed " + path) + + try: + os.makedirs(path) + return True + except OSError as exception: + if exception.errno != errno.EEXIST: + print("ERROR creating", path) + return False + else: + return True + +#---------------------------------------------------------------------------- +def parseStationInfo(filename, index): + + lat = MISSING + lon = MISSING + elev = MISSING + station = "" + + # the file is assumed already downloaded, in subdirectory stations + fname = "./stations/" + filename + + # initialize station values + station = setStationId(fname) + if not station: + return [index, lat, lon, elev, station] + elev = setElev(fname) + lat = setLat(fname) + lon = setLon(fname) + + return [index, lat, lon, elev, station] + +#---------------------------------------------- +def setStationId(fname): + stationId = "" + cmd = 'grep "var currentstnid" ' + fname + s = doCmd(cmd, True) + if s: + index6 = s.find("'", 0) + index7 = s.find("'", index6+1) + stationId = s[index6+1:index7] + return stationId + +#---------------------------------------------- +def setElev(fname): + elev = MISSING + cmd = 'grep "Site elev" ' + fname + #print(cmd) + s = doCmd(cmd) + if s: + if "m above mean sea level" in s: + # scan to + index6 = s.find("") + index7 = s.find("m above") + elev = float(s[index6+4:index7]) + elif " sea level', index_all+1) + if indexend == -1: + print("UNexpected lack of />") + break + + data = data_all[index_all:indexend+2] + + index = 0 + + # expect to see ' Date: Fri, 24 Feb 2023 13:10:13 -0700 Subject: [PATCH 3/9] Redesigned python scrip build_ndbc_stations_from_web.py to use a class --- .../utility/build_ndbc_stations_from_web.py | 457 ++++++++++-------- 1 file changed, 267 insertions(+), 190 deletions(-) diff --git a/scripts/utility/build_ndbc_stations_from_web.py b/scripts/utility/build_ndbc_stations_from_web.py index 19f70b5827..5445a6d509 100755 --- a/scripts/utility/build_ndbc_stations_from_web.py +++ b/scripts/utility/build_ndbc_stations_from_web.py @@ -9,18 +9,23 @@ The list contains latitude, longitude and elevation data for all known stations. The local list can be read by ascii2nc for processing of NDBC data. Algorithm: - Pull down active station xml file from web and create a list of active information. - Optionally write the list to an active stations text file. - + Pull down active station xml file from web and create a list of active information objects. + Write the list to an active stations text file. Pull down complete index list from web. for each file refered to in the complete index list contents: - pull down that stations web page data and append to list of complete station information - optionally write the list of complete station info to a text file. - optionally save all the individual web page data that was pulled down into a subdirectory. + pull down that stations web page data and append to list of complete station information objects. + Write the list of complete station info objects to a text file. + Save all the individual web page data that was pulled down into a subdirectory. + + Warn about stations that are both active and complete but have different values, priority going to TBD. + Count up number of stations that are active but not complete, complete but not active and report. + + Create a master list of all the unique stations from both lists, defer to TBD for any conflicts. + Write the master list. + + Compare the complete list to the previous list of stations in the MET repo, and report on any stations + that have disappeared from the web, or have changed locations. - warn about stations that are both active and complete but have different values - create a master list of all the unique stations from both lists, defer to active for any conflicts - write the master list ''' from optparse import OptionParser @@ -40,20 +45,26 @@ # hardwired website with active station xml ACTIVE_WEBSITE = "https://www.ndbc.noaa.gov/activestations.xml" +# hardwired data subdirectory +DATA_SUBDIR = "/data" + +#hardwired complete stations subdirecdtory +STATIONS_SUBDIR = "/data/stations" + # hardwired result of a wget of ACTIVE_WEBSITE -ACTIVE_STATIONS_XML = "activestations.xml" +ACTIVE_STATIONS_XML = "./data/activestations.xml" # hardwired website with index to a complete list of stations COMPLETE_INDEX_WEBSITE = "https://www.ndbc.noaa.gov/to_station.shtml" # hardwired result of a wget of COMPLETE_INDEX_WEBSITE -COMPLETE_STATIONS_INDEX_INFO = "to_station.shtml" +COMPLETE_STATIONS_INDEX_INFO = "./data/to_station.shtml" # hardwired name of optionally saved active stations -ACTIVE_TEXT_FILE = "active.txt" +ACTIVE_TEXT_FILE = "./data/active.txt" # hardwired name of optionally saved complete stations -COMPLETE_TEXT_FILE = "complete.txt" +COMPLETE_TEXT_FILE = "./data/complete.txt" # default output file name DEFAULT_OUTPUT_FILE = "merged.txt" @@ -61,22 +72,55 @@ MISSING = -99.9 def usage(): - print(f'Usage: BuildNdbcStationsFromWeb.py <--save_complete_stations> <--save_intermediate_lists> <--out=out_filename>') - print(f' -s/--save_intermediate_lists: save intermediate active and complete text files, as well as a subdirectory with all the individual complete stations full information (optional, default: False)"') - print(f' if True files {ACTIVE_TEXT_FILE} and ${COMPLETE_TEXT_FILE} and contents of subdir ./stations are saved, otherwise deleted"') + print(f'Usage: BuildNdbcStationsFromWeb.py , <--diagnostic> <--out=out_filename>') + print(f' -d/--diagnostic: special mode to rerun using already downloaded files, skips all downloading if True') print(f' -o/--out=out_filename: save final text into the named file (default: file name is {DEFAULT_OUTPUT_FILE})"') print(f' Note: <> indicates optional arguments') #---------------------------------------------- def create_parser_options(parser): - parser.add_option("-i", "--save_intermediate_lists", dest="save_intermediate_lists", - action="store_true", default=False, - help=" save intermediate files pulled from web:" + ACTIVE_TEXT_FILE + "," + COMPLETE_TEXT_FILE + "and save individual complete station info pulled from the web into a subdirectory ./stations (optional, default: False)") + parser.add_option("-d", "--diagnostic", dest="diagnostic", action="store_true", default=False, help="Rerun using downlaoded files, skipping download step (optional, default: False)") parser.add_option("-o", "--out", dest="out_file", default=DEFAULT_OUTPUT_FILE, help=" Save the text into the named file (default: " + DEFAULT_OUTPUT_FILE +" )") parser.add_option("-H", "--Help", dest="full_usage", action="store_true", default=False, help = " show more usage information (optional, default = False)") return parser.parse_args() +#---------------------------------------------- +class Station: + def __init__(self, name = "", idvalue="", lat=MISSING, lon=MISSING, elev=MISSING): + self._name = name + self._id = idvalue + self._lat = lat + self._lon = lon + self._elev = elev + + def empty(self): + return self._id == "" + + def textForLookups(self): + txt = ''.format(a=self._id,b=self._lat,c=self._lon,d=self._elev) + return txt + + def location_match(self, other): + if self.empty() or other.empty(): + # this method is used to print mismatches, so don't print mismatches to empty stations + return True + return self._lat == other._lat and self._lon == other._lon and self._elev == other._elev + + def location_string(self): + txt = '{a}({b},{c},{d})'.format(a=self._name,b=self._lat,c=self._lon,d=self._elev) + return txt + + def equals(self, other): + return self._id == other._id and self._lat == other._lat and self._lon == other._lon and self._elev == other._elev + +#---------------------------------------------- +def matchingId(id, stations): + for station in stations: + if station._id == id: + return station + return Station() + #---------------------------------------------- def doCmd(cmd, debug=False): @@ -93,79 +137,146 @@ def doCmd(cmd, debug=False): print("Command failed ", cmd) return "" +#---------------------------------------------------------------------------- +def makeOrScrub(path, debug=False): + if (debug): + print("Recreating path " + path) + if (os.path.exists(path)): + try: + shutil.rmtree(path) + os.makedirs(path) + except: + print('WARNING: ' + path + ' not completely cleaned out.') + else: + os.makedirs(path) + + #---------------------------------------------- -def main(save_intermediate_lists, out_file): +def main(diagnostic, out_file): status = True - # pull the active stations xml from the web - cmd = "wget " + ACTIVE_WEBSITE - print(cmd) - s = doCmd(cmd, True) - if not s: - status = False + cwd = os.getcwd() + + if not diagnostic: + dataDir = cwd + DATA_SUBDIR + print("cleanining out ", dataDir) + makeOrScrub(dataDir) + + os.chdir(dataDir) + + # pull the active stations xml from the web + cmd = "wget " + ACTIVE_WEBSITE + print(cmd) + s = doCmd(cmd, True) + if not s: + status = False + # pull the complete stations html from the web + cmd = "wget " + COMPLETE_INDEX_WEBSITE + print(cmd) + s = doCmd(cmd, True) + if not s: + status = False + if not status: + print("ERROR reading web content") + os.exit(1) + + # move back to top directory + os.chdir(cwd) - # pull the complete stations html from the web - cmd = "wget " + COMPLETE_INDEX_WEBSITE - print(cmd) - s = doCmd(cmd, True) - if not s: - status = False - if not status: - print("ERROR reading web content") - os.exit(1) - # prepare to compare to the default stations file to see what has changed - [ids0, lats0, lons0, elevs0] = parse(DEFAULT_STATIONS_FILE) + default_stations = parse("Default", DEFAULT_STATIONS_FILE) + #[ids_default, lats_default, lons_default, elevs_default] = parse(DEFAULT_STATIONS_FILE) + print("PARSED DEFAUILT STATIONS FILE NUM=", len(default_stations)) - - # parse the active stations XML to create a list - [ids, lats, lons, elevs] = processActive(save_intermediate_lists) + # parse the active stations XML to create a list, which will become the final list + if diagnostic: + active_stations = parse("Active", ACTIVE_TEXT_FILE) + print("PARSED ACTIVE STATION FILES: num=", len(active_stations)) + else: + active_stations = processActive("Active") + print("BUILT ACTIVE STATION FILES: num=", len(active_stations)) # read the complete stations html, find all the individual stations web links, # pull each stations data, parse that downloaded station content to create a list - [ids2, lats2, lons2, elevs2] = processComplete(save_intermediate_lists) + if diagnostic: + complete_stations = parse("Complete", COMPLETE_TEXT_FILE) + #[ids_complete, lats_complete, lons_complete, elevs_complete] = parse(COMPLETE_TEXT_FILE) + print("PARSED COMPLETE STATIONS FILES: num=", len(complete_stations)) + else: + complete_stations = processComplete("Complete") + #[ids_complete, lats_complete, lons_complete, elevs_complete] = processComplete() + print("BUILT COMPLETE STATIONS FILES: num=", len(complete_stations)) # see which ids are not in complete from active, and which have different lat/lons - # note the one used if that happens is always the active one - for id in ids: - i1 = ids.index(id) - if id in ids2: - i2 = ids2.index(id) - if (lats[i1] != lats2[i2]) or (lons[i1] != lons2[i2]) or (elevs[i1] != elevs2[i2]): - print("latlonelev disagree for ", id, " active:(", lats[i1], ",", lons[i1], ",", elevs[i1], - ") complete:(", lats2[i2], ",", lons2[i2], ",", elevs2[i2], ")") - - # see which ids are not in active but are in complete, make a list of those as ones to merge - toMergeIndex = [] - for id in ids2: - i2 = ids2.index(id) - if id in ids: - i1 = ids.index(id) + # note the one used if that happens is always the active one at this point + numConflict = 0 + numActiveNotComplete = 0 + numCompleteNotActive = 0 + numCompleteAndActive = 0 + + for active in active_stations: + id = active._id + complete = matchingId(id, complete_stations) + if complete.empty(): + numActiveNotComplete = numActiveNotComplete+1 else: - toMergeIndex.append(i2) - - print("Merging ", len(toMergeIndex), " items from complete into active") + numCompleteAndActive = numCompleteAndActive+1 + if (not active.location_match(complete)): + numConflict = numConflict + 1 + print("latlonelev disagree for ", id, ":", active.location_string(), ",", complete.location_string()) + + for complete in complete_stations: + id = complete._id + active = matchingId(id, active_stations) + if active.empty(): + numCompleteNotActive = numCompleteNotActive + 1 + + # see which id's have vanished from the current default list + # and which have conflicts with active and/or complete lists + numVanished = 0 + print("Comparing current default stations to final list") + for default in default_stations: + id = default._id + active = matchingId(id, active_stations) + complete = matchingId(id, complete_stations) + if active.empty() and complete.empty(): + print("Station in the local table file but no longer on the webpages:", id) + numVanished = numVanished+1 + else: + if (not active.location_match(default)): + numConflict = numConflict + 1 + print("latlonelev disagree for ", id, ":", active.location_string(), ",", default.location_string()) + if not active.equals(complete): + if (not complete.location_match(default)): + numConflict = numConflict + 1 + print("latlonelev disagree for ", id, ":", complete.location_string(), ",", default.location_string()) - for i in toMergeIndex: - id = ids2[i] - lat = lats2[i] - lon = lons2[i] - elev = elevs2[i] - ids.append(id) - lats.append(lat) - lons.append(lon) - elevs.append(elev) + # see which ids are not in active but are in complete, make a list of those as ones to merge + # Note might add in something about the default lists as well + toMerge = [] + for complete in complete_stations: + id = complete._id + active = matchingId(id, active_stations) + if active.empty(): + toMerge.append(complete) + print("Merging ", len(toMerge), " items from complete into active to make final list") + final_stations = active_stations + for m in toMerge: + final_stations.append(m) + + numNew = 0 + for f in final_stations: + id = f._id + default = matchingId(id, default_stations) + if default.empty(): + print("New station on web not in local table file:", id) + numNew = numNew+1 #now write out the full meal deal by creating a string list nout = 0 txtAll = [] - for id in ids: - i = ids.index(id) - station = id - lat = lats[i] - lon = lons[i] - elev = elevs[i] - txt = ''.format(a=station,b=lat,c=lon,d=elev) + for f in final_stations: + txt = f.textForLookups() txtAll.append(txt) nout = nout + 1 @@ -175,44 +286,34 @@ def main(save_intermediate_lists, out_file): for txt in txtAll: fout.write(txt+"\n") fout.close() - print("Done, wrote out ", nout, " total items to ", out_file) - - - print("Comparing to current default stations") - for id in ids0: - if id not in ids: - print("Station vanished from web:", id) - else: - i0 = ids0.index(id) - station = id - lat0 = lats0[i0] - lon0 = lons0[i0] - elev0 = elevs0[i0] - i = ids.index(id) - lat = lats[i] - lon = lons[i] - elev = elevs[i] - if (lat0 != lat or lon0 != lon or elev0 != elev): - print("latlonelev disagree for ", id, " web:(", lat, ",", lon, ",", elev, - ") default:(", lat0, ",", lon0, ",", elev0, ")") - for id in ids: - if id not in ids0: - print("New station on web not in defaults:", id) + print("Done, wrote out ", nout, " total items to ", out_file) + print("Number of stations that vanished (are in default ndbc_stations.xml and are not now online): ", numVanished) + print("Number of stations that appeared (not in default ndbc_stations.xml and are now online): ", numNew) + print("Number of stations for which there is a conflict from the various sources:", numConflict) + print("Number of stations for which there is both and active and a complete entry:", numCompleteAndActive) + print("Number of stations for which there is an active but no complete entry:", numActiveNotComplete) + print("Number of stations for which there is a complete but no active entry:", numCompleteNotActive) + return 0 #---------------------------------------------------- -def processComplete(save_intermediate): +def processComplete(name): ''' read the complete stations html, find all the individual stations web links, pull each stations data, parse that downloaded station content to create a list ''' - # initialize these lists to empty - ids = [] - lats = [] - lons = [] - elevs = [] + # initialize return to empty + stations = [] + + # create the output location, which should be ./data/stations + cwd = os.getcwd() + outLoc = cwd + STATIONS_SUBDIR + if not makeDirIfNeeded(outLoc): + print("ERROR creating storage for individual station files ", outLoc) + return stations + # Open the file with the list of php pages online (or local files pulled down) with open(COMPLETE_STATIONS_INDEX_INFO, 'r') as file: @@ -224,54 +325,39 @@ def processComplete(save_intermediate): txtAll = [] while index < len(data): # pull down another stations info if you can, and parse it - [index, lat, lon, elev, station] = createNextStationInfo(data, index) + [index, station] = createNextStationInfo(name, data, index) if index == -1: break - - if station: + if not station.empty(): # form a string and append that plus all the individual stuff to lists - txt = ''.format(a=station,b=lat,c=lon,d=elev) + txt = station.textForLookups() txtAll.append(txt) - ids.append(station) - lats.append(lat) - lons.append(lon) - elevs.append(elev) - - if save_intermediate: - # keep the subdirectory of individual stations information + stations.append(station) - # sort the list for ease of use, then write it - txtAll.sort() - fout = open(COMPLETE_TEXT_FILE, "w") - for txt in txtAll: - fout.write(txt+"\n") - fout.close() - - - else: - # delete the subdirectory stations - path = os.getcwd() + "/stations" - if (os.path.exists(path)): - try: - shutil.rmtree(path) - except: - print('WARNING: ' + path + ' not completely cleaned out.') - - return [ids, lats, lons, elevs] + # keep the subdirectory of individual stations information + # sort the list for ease of use, then write it + txtAll.sort() + fout = open(COMPLETE_TEXT_FILE, "w") + for txt in txtAll: + fout.write(txt+"\n") + fout.close() + return stations #---------------------------------------------- -def createNextStationInfo(data, i): +def createNextStationInfo(name, data, i): - lat = MISSING - lon = MISSING - elev = MISSING - station = "" + #lat = MISSING + #lon = MISSING + #elev = MISSING + #station = "" + + s = Station() #data has entries like this: 45001 #on entry i points to the starting location within data to start looking index = data.find('href="station_page.php?', i) if index == -1: - return [-1, lat, lon, elev, station] + return [-1, s] # the stuff beyond 'href="' is the file name that you get via wget, followed by another '"' index2 = index + 6 # point to 'station_page' @@ -285,26 +371,22 @@ def createNextStationInfo(data, i): # name of returned file filename = data[index2:index3] + # temporarily change to the correct subdirectory + cwd = os.getcwd() + os.chdir(cwd + STATIONS_SUBDIR) # go get it cmd = 'wget "' + ref + '"' print(cmd) s = doCmd(cmd, True) + # move back + os.chdir(cwd) if not s: # note try to keep going forward as index has been updated print("ERROR data not online: ", ref) - return [index, lat, lon, elev, station] + return [index, s] - # make a stations subdirectory if needed - cwd = os.getcwd() - if not makeDirIfNeeded(cwd + "/stations"): - print("ERROR cannot create needed subdirectory 'stations'") - return [index, lat, lon, elev, station] - - # move this file to the stations subdirectory - shutil.move(cwd + "/" + filename, cwd + "/stations/" + filename) - # parse the file and return the information, including the next index - return parseStationInfo(filename, index) + return parseStationInfo(name, cwd + STATIONS_SUBDIR + "/" + filename, index) #---------------------------------------------------------------------------- def makeDirIfNeeded(path, debug=False): @@ -322,25 +404,20 @@ def makeDirIfNeeded(path, debug=False): return True #---------------------------------------------------------------------------- -def parseStationInfo(filename, index): - - lat = MISSING - lon = MISSING - elev = MISSING - station = "" +def parseStationInfo(name, fname, index): - # the file is assumed already downloaded, in subdirectory stations - fname = "./stations/" + filename + s = Station() + # the file is assumed already downloaded # initialize station values station = setStationId(fname) if not station: - return [index, lat, lon, elev, station] + return [index, s] elev = setElev(fname) lat = setLat(fname) lon = setLon(fname) - - return [index, lat, lon, elev, station] + s = Station(name, station, lat, lon, elev) + return [index, s] #---------------------------------------------- def setStationId(fname): @@ -400,36 +477,32 @@ def setLon(fname): return lon #---------------------------------------------- -def processActive(save_intermediate): +def processActive(name): ''' read the active stations XML to create a list ''' - [ids, lats, lons, elevs] = parse(ACTIVE_STATIONS_XML) + astations = parse(name, ACTIVE_STATIONS_XML) txtAll = [] - for id in ids: - i = ids.index(id) - station = ids[i] - lat = lats[i] - lon = lons[i] - elev = elevs[i] - txt = ''.format(a=station,b=lat,c=lon,d=elev) + for s in astations: + txt = s.textForLookups() txtAll.append(txt) txtAll.sort() - if save_intermediate: - fout = open(ACTIVE_TEXT_FILE, "w") - for txt in txtAll: - fout.write(txt+"\n") - fout.close() - - return [ids, lats, lons, elevs] + fout = open(ACTIVE_TEXT_FILE, "w") + for txt in txtAll: + fout.write(txt+"\n") + fout.close() + return astations #---------------------------------------------- -def parse(fname): - ids = [] - lats = [] - lons = [] - elevs = [] +def parse(name, fname, debug=False): + if debug: + print("Parsing ", fname) + stations = [] + #ids = [] + #lats = [] + #lons = [] + #elevs = [] with open(fname, 'r') as file: data_all = file.read().replace('\n', '') @@ -445,7 +518,8 @@ def parse(fname): break data = data_all[index_all:indexend+2] - + if debug: + print("Parsing this: ", data) index = 0 # expect to see ' Date: Fri, 24 Feb 2023 16:38:24 -0700 Subject: [PATCH 4/9] Implemented the approach suggested by Gwen Chen NOAA --- .../utility/build_ndbc_stations_from_web.py | 218 ++++++++++++------ 1 file changed, 146 insertions(+), 72 deletions(-) diff --git a/scripts/utility/build_ndbc_stations_from_web.py b/scripts/utility/build_ndbc_stations_from_web.py index 5445a6d509..e785d217b8 100755 --- a/scripts/utility/build_ndbc_stations_from_web.py +++ b/scripts/utility/build_ndbc_stations_from_web.py @@ -7,8 +7,9 @@ The script reads NDBC station information from two NOAA websites and merges the contents into one local list. The list contains latitude, longitude and elevation data for all known stations. -The local list can be read by ascii2nc for processing of NDBC data. +The local list can be read by ascii2nc for processing of NDBC data inputs. Algorithm: + Read the current default ndbc_stations.xml file and create a list of default station information objects. Pull down active station xml file from web and create a list of active information objects. Write the list to an active stations text file. Pull down complete index list from web. @@ -17,14 +18,22 @@ Write the list of complete station info objects to a text file. Save all the individual web page data that was pulled down into a subdirectory. - Warn about stations that are both active and complete but have different values, priority going to TBD. - Count up number of stations that are active but not complete, complete but not active and report. + Compare the complete stations information to the default station information objects. + If a station is on the complete list but not on the default list, add it to the default list. + If a station is on both lists, but has different location info, change the locaiton info to that of the complete, + (unless the complete has no meaningful lat/lon information, typically 0,0). - Create a master list of all the unique stations from both lists, defer to TBD for any conflicts. - Write the master list. + Compare the augmented default list to the active stations list. + If a station is on the active list but not on the default list, add it to the default list. + If a station is on both lists, but has different location info, keep the default list values + (unless the default has no meaningful lat/lon information, typically 0,0, then change to the active). - Compare the complete list to the previous list of stations in the MET repo, and report on any stations - that have disappeared from the web, or have changed locations. + Log Warnings about discrepancies. + Keep counts of everything. + + Write the final default list a a new output. + + Optionally prune the default list, removing all stations that are not active or complete. ''' @@ -46,25 +55,25 @@ ACTIVE_WEBSITE = "https://www.ndbc.noaa.gov/activestations.xml" # hardwired data subdirectory -DATA_SUBDIR = "/data" +DATA_SUBDIR = "/ndbc_temp_data" #hardwired complete stations subdirecdtory -STATIONS_SUBDIR = "/data/stations" +STATIONS_SUBDIR = "/ndbc_temp_data/stations" # hardwired result of a wget of ACTIVE_WEBSITE -ACTIVE_STATIONS_XML = "./data/activestations.xml" +ACTIVE_STATIONS_XML = "./ndbc_temp_data/activestations.xml" # hardwired website with index to a complete list of stations COMPLETE_INDEX_WEBSITE = "https://www.ndbc.noaa.gov/to_station.shtml" # hardwired result of a wget of COMPLETE_INDEX_WEBSITE -COMPLETE_STATIONS_INDEX_INFO = "./data/to_station.shtml" +COMPLETE_STATIONS_INDEX_INFO = "./ndbc_temp_data/to_station.shtml" # hardwired name of optionally saved active stations -ACTIVE_TEXT_FILE = "./data/active.txt" +ACTIVE_TEXT_FILE = "./ndbc_temp_data/active.txt" # hardwired name of optionally saved complete stations -COMPLETE_TEXT_FILE = "./data/complete.txt" +COMPLETE_TEXT_FILE = "./ndbc_temp_data/complete.txt" # default output file name DEFAULT_OUTPUT_FILE = "merged.txt" @@ -72,14 +81,16 @@ MISSING = -99.9 def usage(): - print(f'Usage: BuildNdbcStationsFromWeb.py , <--diagnostic> <--out=out_filename>') - print(f' -d/--diagnostic: special mode to rerun using already downloaded files, skips all downloading if True') + print(f'Usage: BuildNdbcStationsFromWeb.py , <--diagnostic> <--out=out_filename> <--prune>') + print(f' -d/--diagnostic: special mode to rerun using already downloaded files, skips all downloading if True (Downloaded files are in ./{DATA_SUBDIR}') print(f' -o/--out=out_filename: save final text into the named file (default: file name is {DEFAULT_OUTPUT_FILE})"') + print(f' -p/--prune: Delete all stations from the local ndbc_stations file that are no longer online') print(f' Note: <> indicates optional arguments') #---------------------------------------------- def create_parser_options(parser): parser.add_option("-d", "--diagnostic", dest="diagnostic", action="store_true", default=False, help="Rerun using downlaoded files, skipping download step (optional, default: False)") + parser.add_option("-p", "--prune", dest="prune", action="store_true", default=False, help="Prune files that are no longer online (optional, default:False)") parser.add_option("-o", "--out", dest="out_file", default=DEFAULT_OUTPUT_FILE, help=" Save the text into the named file (default: " + DEFAULT_OUTPUT_FILE +" )") parser.add_option("-H", "--Help", dest="full_usage", action="store_true", default=False, help = " show more usage information (optional, default = False)") @@ -113,7 +124,44 @@ def location_string(self): def equals(self, other): return self._id == other._id and self._lat == other._lat and self._lon == other._lon and self._elev == other._elev + + def setName(self, name): + self._name = name +#---------------------------------------------- +def replaceLatLonIfGood(header, name, stations, station): + if station._lat == 0 and station._lon == 0: + #print(header, ",No replacement using:", station.textForLookups()) + return False + for n in range(len(stations)): + if stations[n]._id == station._id: + print(header, "Replacing: ", stations[n].textForLookups(), " with ", station.textForLookups()) + s = station + s.setName(name) + stations[n] = station + return True + print("Warning:", header, "No match for replacment of station ", station._id) + return False + +#---------------------------------------------- +def replaceLatLonIfListIsBad(header, name, stations, station): + if station._lat == 0 and station._lon == 0: + #print(header, ",No replacement using:", station.textForLookups()) + return False + for n in range(len(stations)): + if stations[n]._id == station._id: + if stations[n]._lat == 0 and stations[n]._lon == 0: + print(header, "Replacing: ", stations[n].textForLookups(), " with ", station.textForLookups()) + s = station + s.setName(name) + stations[n] = station + return True + else: + return False + + print("Warning:", header, "No match for replacment of station ", station._id) + return False + #---------------------------------------------- def matchingId(id, stations): for station in stations: @@ -152,12 +200,13 @@ def makeOrScrub(path, debug=False): #---------------------------------------------- -def main(diagnostic, out_file): - status = True +def main(diagnostic, out_file, prune): cwd = os.getcwd() if not diagnostic: + status = True + dataDir = cwd + DATA_SUBDIR print("cleanining out ", dataDir) makeOrScrub(dataDir) @@ -185,8 +234,13 @@ def main(diagnostic, out_file): # prepare to compare to the default stations file to see what has changed default_stations = parse("Default", DEFAULT_STATIONS_FILE) - #[ids_default, lats_default, lons_default, elevs_default] = parse(DEFAULT_STATIONS_FILE) + numDefault = len(default_stations) print("PARSED DEFAUILT STATIONS FILE NUM=", len(default_stations)) + + # make a copy of this as the final outputs + final_stations = default_stations + for f in final_stations: + f.setName("Final") # parse the active stations XML to create a list, which will become the final list if diagnostic: @@ -200,85 +254,99 @@ def main(diagnostic, out_file): # pull each stations data, parse that downloaded station content to create a list if diagnostic: complete_stations = parse("Complete", COMPLETE_TEXT_FILE) - #[ids_complete, lats_complete, lons_complete, elevs_complete] = parse(COMPLETE_TEXT_FILE) print("PARSED COMPLETE STATIONS FILES: num=", len(complete_stations)) else: complete_stations = processComplete("Complete") - #[ids_complete, lats_complete, lons_complete, elevs_complete] = processComplete() print("BUILT COMPLETE STATIONS FILES: num=", len(complete_stations)) # see which ids are not in complete from active, and which have different lat/lons # note the one used if that happens is always the active one at this point + numNew = 0 + numNewComplete = 0 + numNewActive = 0 numConflict = 0 - numActiveNotComplete = 0 + numConflictChanged = 0 + numComplete = 0 + numActive = 0 numCompleteNotActive = 0 - numCompleteAndActive = 0 + numActiveNotComplete = 0 + # compare complete stations to default stations + for complete in complete_stations: + numComplete = numComplete + 1 + id = complete._id + default = matchingId(id, default_stations) + active = matchingId(id, active_stations) + if active.empty(): + numCompleteNotActive = numCompleteNotActive + 1 + if default.empty(): + # station is on the complete list but not on the default list, add it + f = complete + f.setName("Final") + final_stations.append(f) + numNew = numNew+1 + numNewComplete = numNewComplete + 1 + else: + # compare complete and default + if not complete.location_match(default): + numConflict = numConflict + 1 + if replaceLatLonIfGood("Complete to Final", "Final", final_stations, complete): + numConflictChanged = numConflictChanged + 1 + + # compare active stations to final stations for active in active_stations: + numActive = numActive + 1 id = active._id + final = matchingId(id, final_stations) complete = matchingId(id, complete_stations) if complete.empty(): - numActiveNotComplete = numActiveNotComplete+1 + numActiveNotComplete = numActiveNotComplete +1 + if final.empty(): + # station is on the active list but not on the final list, add it + a = active + a.setName("Final") + final_stations.append(a) + numNew = numNew+1 + numNewActive = numNewActive + 1 else: - numCompleteAndActive = numCompleteAndActive+1 - if (not active.location_match(complete)): + # compare complete and default + if not final.location_match(active): numConflict = numConflict + 1 - print("latlonelev disagree for ", id, ":", active.location_string(), ",", complete.location_string()) - - for complete in complete_stations: - id = complete._id - active = matchingId(id, active_stations) - if active.empty(): - numCompleteNotActive = numCompleteNotActive + 1 - - # see which id's have vanished from the current default list - # and which have conflicts with active and/or complete lists + if replaceLatLonIfListIsBad("Active to Final", "Final", final_stations, active): + numConflictChanged = numConflictChanged + 1 + + # see which id's have vanished from the current default list, to be used when prune is true numVanished = 0 + purgeIds = [] print("Comparing current default stations to final list") for default in default_stations: id = default._id active = matchingId(id, active_stations) complete = matchingId(id, complete_stations) if active.empty() and complete.empty(): - print("Station in the local table file but no longer on the webpages:", id) + #print("Station in the local table file but no longer on the webpages:", id) numVanished = numVanished+1 - else: - if (not active.location_match(default)): - numConflict = numConflict + 1 - print("latlonelev disagree for ", id, ":", active.location_string(), ",", default.location_string()) - if not active.equals(complete): - if (not complete.location_match(default)): - numConflict = numConflict + 1 - print("latlonelev disagree for ", id, ":", complete.location_string(), ",", default.location_string()) - - # see which ids are not in active but are in complete, make a list of those as ones to merge - # Note might add in something about the default lists as well - toMerge = [] - for complete in complete_stations: - id = complete._id - active = matchingId(id, active_stations) - if active.empty(): - toMerge.append(complete) - print("Merging ", len(toMerge), " items from complete into active to make final list") - final_stations = active_stations - for m in toMerge: - final_stations.append(m) + purgeIds.append(id) - numNew = 0 for f in final_stations: id = f._id default = matchingId(id, default_stations) if default.empty(): - print("New station on web not in local table file:", id) + #print("New station on web not in local table file:", id) numNew = numNew+1 #now write out the full meal deal by creating a string list nout = 0 + nprune = 0 txtAll = [] for f in final_stations: - txt = f.textForLookups() - txtAll.append(txt) - nout = nout + 1 + if prune and f.IdOnList(purgeIds): + print("Pruning station: ", f._id, " No longer on line") + nprune = nprune + 1 + else: + txt = f.textForLookups() + txtAll.append(txt) + nout = nout + 1 # sort for ease of use txtAll.sort() @@ -287,14 +355,20 @@ def main(diagnostic, out_file): fout.write(txt+"\n") fout.close() - print("Done, wrote out ", nout, " total items to ", out_file) - print("Number of stations that vanished (are in default ndbc_stations.xml and are not now online): ", numVanished) - print("Number of stations that appeared (not in default ndbc_stations.xml and are now online): ", numNew) - print("Number of stations for which there is a conflict from the various sources:", numConflict) - print("Number of stations for which there is both and active and a complete entry:", numCompleteAndActive) - print("Number of stations for which there is an active but no complete entry:", numActiveNotComplete) - print("Number of stations for which there is a complete but no active entry:", numCompleteNotActive) - + print("Num complete: ", numComplete) + print("Num active: ", numActive) + print("Num default: ", numDefault) + print("Num final: ", nout) + print("Num pruned: ", nprune) + print("Num vanished: ", numVanished) + print("Num new complete: ", numNewComplete) + print("Num new active: ", numNewActive) + print("Num new total: ", numNew) + print("Num conflict no change: ", numConflict) + print("Num conflict with change:", numConflictChanged) + print("Numactivenotcomplete: ", numActiveNotComplete) + print("Numcompletenotactive: ", numCompleteNotActive) + return 0 #---------------------------------------------------- @@ -307,7 +381,7 @@ def processComplete(name): # initialize return to empty stations = [] - # create the output location, which should be ./data/stations + # create the output location, which should be ./ndbc_temp_data/stations cwd = os.getcwd() outLoc = cwd + STATIONS_SUBDIR if not makeDirIfNeeded(outLoc): @@ -564,5 +638,5 @@ def parse(name, fname, debug=False): if options.full_usage: usage() exit(0) - main(options.diagnostic, options.out_file) + main(options.diagnostic, options.out_file, options.prune) From a975a77b7a247cb1d0f2578719f24eab2abe06ac Mon Sep 17 00:00:00 2001 From: David Albo Date: Mon, 27 Feb 2023 14:39:05 -0700 Subject: [PATCH 5/9] Added new NDBC stations updating script to make --- scripts/utility/Makefile.am | 3 ++- scripts/utility/Makefile.in | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/utility/Makefile.am b/scripts/utility/Makefile.am index 6e22a33e2b..d807a69977 100644 --- a/scripts/utility/Makefile.am +++ b/scripts/utility/Makefile.am @@ -26,7 +26,8 @@ pythonutilitydir = $(pkgdatadir)/utility pythonutility_DATA = \ - print_pointnc2ascii.py + print_pointnc2ascii.py \ + build_ndbc_stations_from_web.py EXTRA_DIST = ${pythonutility_DATA} diff --git a/scripts/utility/Makefile.in b/scripts/utility/Makefile.in index a515a31201..bdaec7b3f9 100644 --- a/scripts/utility/Makefile.in +++ b/scripts/utility/Makefile.in @@ -298,7 +298,8 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ pythonutilitydir = $(pkgdatadir)/utility pythonutility_DATA = \ - print_pointnc2ascii.py + print_pointnc2ascii.py \ + build_ndbc_stations_from_web.py EXTRA_DIST = ${pythonutility_DATA} MAINTAINERCLEANFILES = Makefile.in From dcb1494a38daad84c0b394d72452568666918215 Mon Sep 17 00:00:00 2001 From: David Albo Date: Mon, 27 Feb 2023 14:40:25 -0700 Subject: [PATCH 6/9] Updated ndbc_stations.xml file --- data/table_files/ndbc_stations.xml | 3993 ++++++++++++++++++---------- 1 file changed, 2613 insertions(+), 1380 deletions(-) diff --git a/data/table_files/ndbc_stations.xml b/data/table_files/ndbc_stations.xml index c536e399f4..cacaa889e2 100644 --- a/data/table_files/ndbc_stations.xml +++ b/data/table_files/ndbc_stations.xml @@ -1,1380 +1,2613 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From c3d5c89f4f3b90ba65ad8cdcbdbf5e34b302942e Mon Sep 17 00:00:00 2001 From: David Albo Date: Mon, 27 Feb 2023 14:42:13 -0700 Subject: [PATCH 7/9] Added documentation about the NDBC stations update script option --- docs/Users_Guide/config_options.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst index 66c2926af0..7e4838d27d 100644 --- a/docs/Users_Guide/config_options.rst +++ b/docs/Users_Guide/config_options.rst @@ -272,6 +272,34 @@ The default table can be found in the installed XML content for all stations that allows lookups of latitude, longitude, and, in some cases, elevation for all stations based on stationId. +This set of stations comes from 2 online sources +`The active stations website `_ +and `The complete stations website `_. +As these lists can change as a function of time, a script can be run to pull +down the contents of both websites and merge any changes with the existing stations +file content, creating an updated stations file locally. +The MET_NDBC_STATIONS environment variable can be then set to refer to this newer +stations file. Also, the MET development team will periodically +run this script and update *share/met/table_files/ndbc_stations.xml*. + +To run this utility: + +build_ndbc_stations_from_web.py <-d> <-p> <-o OUTPUT_FILE> +Usage: build_ndbc_stations_from_web.py [options] +Options: + -h, --help show this help message and exit + -d, --diagnostic Rerun using downlaoded files, skipping download step + (optional, default: False) + -p, --prune Prune files that are no longer online (optional, + default:False) + -o OUT_FILE, --out=OUT_FILE + Save the text into the named file (default: + merged.txt ) + +NOTE: The downloaded files are written to a subdirectory ndbc_temp_data which +can be deleted once the final output file is created. + + MET_BASE ^^^^^^^^ From 9634cc9f03c0e8b1e666259353a62d92949a52f1 Mon Sep 17 00:00:00 2001 From: David Albo Date: Tue, 28 Feb 2023 11:58:17 -0700 Subject: [PATCH 8/9] Changed to not write out an elevation when input elevation is not found --- data/table_files/ndbc_stations.xml | 1498 ++++++++--------- .../utility/build_ndbc_stations_from_web.py | 24 +- 2 files changed, 757 insertions(+), 765 deletions(-) diff --git a/data/table_files/ndbc_stations.xml b/data/table_files/ndbc_stations.xml index cacaa889e2..114957cd5a 100644 --- a/data/table_files/ndbc_stations.xml +++ b/data/table_files/ndbc_stations.xml @@ -1,32 +1,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -43,44 +43,44 @@ - + - - - - - - + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + @@ -97,22 +97,22 @@ - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + @@ -120,18 +120,18 @@ - - - + + + - - - + + + - - - - + + + + @@ -146,54 +146,54 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - - - + + + - + - - - - - - - + + + + + + + - + - + - + @@ -218,11 +218,11 @@ - - - - - + + + + + @@ -234,27 +234,27 @@ - + - - + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -262,38 +262,38 @@ - - - + + + - - - - + + + + - - - - + + + + - + - + - + @@ -303,8 +303,8 @@ - - + + @@ -323,51 +323,51 @@ - + - - - - + + + + - + - + - - + + - + - + - + - + - + - + - + - + @@ -383,11 +383,11 @@ - - - + + + - + @@ -399,7 +399,7 @@ - + @@ -432,13 +432,13 @@ - + - + @@ -448,7 +448,7 @@ - + @@ -457,9 +457,9 @@ - + - + @@ -498,9 +498,9 @@ - + - + @@ -544,53 +544,53 @@ - + - - - - + + + + - - - - - - + + + + + + - + - - - - - + + + + + - - - - + + + + - + - + - + - + @@ -608,7 +608,7 @@ - + @@ -618,7 +618,7 @@ - + @@ -631,15 +631,15 @@ - + - - - - - + + + + + - + @@ -656,7 +656,7 @@ - + @@ -693,11 +693,11 @@ - - + + - + @@ -729,48 +729,48 @@ - + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -779,24 +779,24 @@ - - - - - - - - - - - - + + + + + + + + + + + + - + - + @@ -813,14 +813,14 @@ - - + + - + @@ -845,11 +845,11 @@ - - - - - + + + + + @@ -858,7 +858,7 @@ - + @@ -872,60 +872,60 @@ - - + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - + + + + + + + + - + - - - + + + @@ -937,67 +937,67 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - + - + @@ -1010,18 +1010,18 @@ - + - + - + @@ -1030,18 +1030,18 @@ - + - + - + - + @@ -1052,11 +1052,11 @@ - + - + @@ -1068,24 +1068,24 @@ - + - + - + - + - + @@ -1093,37 +1093,37 @@ - + - + - + - + - + - + - + - + @@ -1137,7 +1137,7 @@ - + @@ -1146,17 +1146,17 @@ - + - - + + - + - + @@ -1165,8 +1165,8 @@ - - + + @@ -1181,47 +1181,47 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - - + + - + - + - + - + - + - + @@ -1233,16 +1233,16 @@ - - - + + + - + - - + + @@ -1253,53 +1253,53 @@ - - + + - - + + - + - + - - + + - + - - - + + + - - - + + + - + - + - + - + - - + + @@ -1307,16 +1307,16 @@ - - + + - + - + - + @@ -1328,85 +1328,85 @@ - + - + - - + + - - - - - - - - + + + + + + + + - + - - - + + + - + - + - - - + + + - - - - + + + + - - - + + + - - - + + + - + - + - + - - - - + + + + - - - - - - + + + + + + - - + + - + @@ -1421,20 +1421,20 @@ - - + + - + - + @@ -1445,7 +1445,7 @@ - + @@ -1453,7 +1453,7 @@ - + @@ -1461,7 +1461,7 @@ - + @@ -1473,7 +1473,7 @@ - + @@ -1485,14 +1485,14 @@ - + - - - + + + @@ -1501,36 +1501,36 @@ - - - + + + - + - + - + - + - + - + @@ -1539,15 +1539,15 @@ - + - + - + @@ -1557,39 +1557,39 @@ - + - + - + - + - + - + - - + + - + @@ -1598,11 +1598,11 @@ - - + + - - + + @@ -1612,7 +1612,7 @@ - + @@ -1624,14 +1624,14 @@ - + - + - + - + @@ -1640,11 +1640,11 @@ - + - + @@ -1653,12 +1653,12 @@ - + - + @@ -1667,11 +1667,11 @@ - + - + @@ -1683,23 +1683,23 @@ - - + + - + - + - - - - + + + + @@ -1709,10 +1709,10 @@ - + - + @@ -1725,39 +1725,39 @@ - + - + - + - + - + - + - + - + - - + + @@ -1765,7 +1765,7 @@ - + @@ -1781,27 +1781,27 @@ - + - + - + - + - + - + - + @@ -1832,35 +1832,35 @@ - - + + - + - + - - - + + + - + - + - + @@ -1869,25 +1869,25 @@ - + - - + + - + - + - - + + - + @@ -1896,36 +1896,36 @@ - + - + - + - + - + - + - + @@ -1935,11 +1935,11 @@ - + - + @@ -1949,17 +1949,17 @@ - + - + - + @@ -1977,24 +1977,24 @@ - + - + - + - + - + @@ -2007,30 +2007,30 @@ - + - + - - + + - + - - + + @@ -2041,14 +2041,14 @@ - + - + - - + + @@ -2056,10 +2056,10 @@ - + - + @@ -2069,10 +2069,10 @@ - - + + - + @@ -2085,39 +2085,39 @@ - + - - + + - + - + - + - + - + - + - + - - + + @@ -2128,9 +2128,9 @@ - + - + @@ -2140,74 +2140,74 @@ - + - + - - + + - - + + - - - + + + - + - - - + + + - + - + - - - + + + - - - - + + + + - - + + - + - + - + - - - + + + - - - - - + + + + + - - + + @@ -2221,7 +2221,7 @@ - + @@ -2239,14 +2239,14 @@ - + - + @@ -2254,7 +2254,7 @@ - + @@ -2265,7 +2265,7 @@ - + @@ -2279,8 +2279,8 @@ - - + + @@ -2292,25 +2292,25 @@ - + - + - + - + - + - + @@ -2318,14 +2318,14 @@ - + - + - + @@ -2341,29 +2341,29 @@ - + - + - + - + - - + + - + @@ -2372,11 +2372,11 @@ - - + + - - + + @@ -2396,12 +2396,12 @@ - + - + - + @@ -2411,21 +2411,21 @@ - + - + - + - + @@ -2436,7 +2436,7 @@ - + @@ -2447,20 +2447,20 @@ - - + + - + - - - + + + @@ -2468,10 +2468,10 @@ - + - + @@ -2490,21 +2490,21 @@ - + - + - - - + + + @@ -2523,23 +2523,23 @@ - + - + - + - + - + - + @@ -2568,11 +2568,11 @@ - + - + @@ -2580,17 +2580,17 @@ - - + + - + - + @@ -2601,13 +2601,13 @@ - + - - + + - + - + diff --git a/scripts/utility/build_ndbc_stations_from_web.py b/scripts/utility/build_ndbc_stations_from_web.py index e785d217b8..3843c05112 100755 --- a/scripts/utility/build_ndbc_stations_from_web.py +++ b/scripts/utility/build_ndbc_stations_from_web.py @@ -109,7 +109,10 @@ def empty(self): return self._id == "" def textForLookups(self): - txt = ''.format(a=self._id,b=self._lat,c=self._lon,d=self._elev) + if self._elev == MISSING: + txt = ''.format(a=self._id,b=self._lat,c=self._lon) + else: + txt = ''.format(a=self._id,b=self._lat,c=self._lon,d=self._elev) return txt def location_match(self, other): @@ -119,7 +122,10 @@ def location_match(self, other): return self._lat == other._lat and self._lon == other._lon and self._elev == other._elev def location_string(self): - txt = '{a}({b},{c},{d})'.format(a=self._name,b=self._lat,c=self._lon,d=self._elev) + if self._elev == MISSING: + txt = '{a}({b},{c})'.format(a=self._name,b=self._lat,c=self._lon) + else: + txt = '{a}({b},{c},{d})'.format(a=self._name,b=self._lat,c=self._lon,d=self._elev) return txt def equals(self, other): @@ -420,11 +426,6 @@ def processComplete(name): #---------------------------------------------- def createNextStationInfo(name, data, i): - #lat = MISSING - #lon = MISSING - #elev = MISSING - #station = "" - s = Station() #data has entries like this: 45001 @@ -573,10 +574,6 @@ def parse(name, fname, debug=False): if debug: print("Parsing ", fname) stations = [] - #ids = [] - #lats = [] - #lons = [] - #elevs = [] with open(fname, 'r') as file: data_all = file.read().replace('\n', '') @@ -619,14 +616,9 @@ def parse(name, fname, debug=False): elev = float(data[index7+6:index8]) stations.append(Station(name, stationId, lat, lon, elev)) - #ids.append(stationId) - #lats.append(lat) - #lons.append(lon) - #elevs.append(elev) index_all = indexend+2 - #return [ids, lats, lons, elevs] return stations #---------------------------------------------- From c6baac3c649203635ff5aa503dce8b83868fc3db Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 28 Feb 2023 13:43:50 -0700 Subject: [PATCH 9/9] Per #2426, adding some spaces into log messages. --- scripts/utility/build_ndbc_stations_from_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/utility/build_ndbc_stations_from_web.py b/scripts/utility/build_ndbc_stations_from_web.py index 3843c05112..cc20b6d02d 100755 --- a/scripts/utility/build_ndbc_stations_from_web.py +++ b/scripts/utility/build_ndbc_stations_from_web.py @@ -372,8 +372,8 @@ def main(diagnostic, out_file, prune): print("Num new total: ", numNew) print("Num conflict no change: ", numConflict) print("Num conflict with change:", numConflictChanged) - print("Numactivenotcomplete: ", numActiveNotComplete) - print("Numcompletenotactive: ", numCompleteNotActive) + print("Num active not complete: ", numActiveNotComplete) + print("Num complete not active: ", numCompleteNotActive) return 0