Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bugfix 2370 main_v10.1 - 6 line headers and yyyy-MMM #2397

Merged
merged 3 commits into from
Jan 10, 2023
Merged

Bugfix 2370 main_v10.1 - 6 line headers and yyyy-MMM #2397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a5e227
#2370 Added month_name_to_m
Dec 22, 2022
f4abd58
#2370 Added a n argument date_offset to _getValidTime
Dec 22, 2022
80c4727
#2370 Support 6 line headers and yyyy-MMM date string for AERONET V3.…
Jan 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions met/src/basic/vx_cal/unix_to_mdyhms.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ using namespace std;

#include <iostream>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <cmath>

Expand Down Expand Up @@ -94,6 +95,35 @@ const char * short_month_name[] = {
};


////////////////////////////////////////////////////////////////////////
// Converts a month string to month (1 to 12).

int month_name_to_m (const char *month_str)

{

int month = -1;
int str_len = strlen(month_str);
char t_month_str[str_len + 1];

for (int idx=0; idx<str_len; idx++) {
t_month_str[idx] = (idx==0) ? toupper(month_str[idx]) : tolower(month_str[idx]);
}
t_month_str[str_len] = 0;

for (int idx=1; idx<=12; idx++) {
if (0 == strcmp(short_month_name[idx], t_month_str)
|| 0 == strcmp(month_name[idx], t_month_str)) {
month = idx;
break;
}
}

return month;

}


////////////////////////////////////////////////////////////////////////


Expand Down
1 change: 1 addition & 0 deletions met/src/basic/vx_cal/vx_cal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ extern unixtime doyhms_to_unix (int doy, int year, int hour, int minute, int
extern unixtime mdyhms_to_unix (int month, int day, int year, int hour, int minute, int second);

extern void unix_to_mdyhms (unixtime u, int & month, int & day, int & year, int & hour, int & minute, int & second);
extern int month_name_to_m (const char *month_str);


extern int date_to_mjd (int m, int d, int y);
Expand Down
117 changes: 67 additions & 50 deletions met/src/tools/other/ascii2nc/aeronet_handler.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ using namespace std;

#include "aeronet_handler.h"

static bool test_AOD_550 = false;
static const char *AERONET_NA_STR = "N/A";
static const char *AERONET_V3_STR = "AERONET Version 3";

const int AeronetHandler::NUM_HDR_COLS = 7;
const int AeronetHandler::NUM_OBS_COLS = 45;
Expand All @@ -46,6 +48,8 @@ const string elv_col1 = "Site_Elevation"; // "Site_Elevation(m)";
const string lat_col2 = "Latitude"; // "Latitude(degrees)"
const string lon_col2 = "Longitude"; // "Longitude(degrees)"
const string elv_col2 = "Elevation"; // "Elevation(meters)"
const string date_col = "Date"; // "Date(dd:mm:yyyy)"
const string month_col = "Month"; // "Month"

const string AeronetHandler::HEADER_TYPE = ""; /////

Expand All @@ -67,6 +71,7 @@ const string WAVELENGTHS_PW_NAME = "Exact_Wavelengths_of_PW"; // Exact_Wavelen
const string WAVELENGTHS_INPUT_AOD_NAME = "Exact_Wavelengths_for_Input_AOD"; // Exact_Wavelengths_for_Input_AOD(um)

static int format_version;
const string SITE_MISSING = "site_missing";

const float AERONET_MISSING_VALUE = -999.;

Expand Down Expand Up @@ -121,7 +126,7 @@ bool AeronetHandler::isFileType(LineDataFile &ascii_file) const
string line = dl.get_line();
if (line.length() > 17) {
line = line.substr(0, 17);
if (strcmp(line.c_str(), "AERONET Version 3") == 0) {
if (strcmp(line.c_str(), AERONET_V3_STR) == 0) {
is_file_type = true;
format_version = 3;
return is_file_type;
Expand Down Expand Up @@ -178,8 +183,9 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)

if (format_version == 3) {
use_var_id = true;
// Get the field information from the 7-th header line
ascii_file >> data_line;
// Skip lines to get the field information from 7-th header line
// (with site name at line 2) or the 6-th header line (without site name)
if(_stationId != SITE_MISSING) ascii_file >> data_line;
ascii_file >> data_line;
ascii_file >> data_line;
}
Expand All @@ -197,7 +203,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)

int flag;
int aod_var_id = bad_data_int;
int var_idx, sid_idx, elv_idx, lat_idx, lon_idx;
int var_idx, sid_idx, elv_idx, lat_idx, lon_idx, date_idx, month_idx;
double height_from_header;
string aot = "AOT";
//string angstrom = "Angstrom";
Expand All @@ -207,7 +213,8 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
IntArray header_var_index;
StringArray header_var_names;

sid_idx = elv_idx = lat_idx = lon_idx = -1;
date_idx = 0;
sid_idx = elv_idx = lat_idx = lon_idx = month_idx = -1;

for (int j = 0; j < hdr_tokens.n_elements(); j++)
{
Expand Down Expand Up @@ -249,6 +256,8 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
else if (0 == hdr_field.find(lat_col1) || 0 == hdr_field.find(lat_col2)) lat_idx = j;
else if (0 == hdr_field.find(lon_col1) || 0 == hdr_field.find(lon_col2)) lon_idx = j;
else if (0 == hdr_field.find(elv_col1) || 0 == hdr_field.find(elv_col2)) elv_idx = j;
else if (0 == strcmp(hdr_field.c_str(), month_col.c_str())) month_idx = j;
else if (0 == hdr_field.find(date_col)) date_idx = j;

// Collect variable names and index
var_name = make_var_name_from_header(hdr_field);
Expand All @@ -263,7 +272,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
header_var_index.add(var_idx);
header_var_names.add(var_name.c_str());
header_heights.add(height_from_header);
mlog << Debug(5) << method_name << "header_idx: " << j
mlog << Debug(7) << method_name << "header_idx: " << j
<< ", var_idx: " << var_idx << ", var: " << var_name << " from " << hdr_field
<< ", flag: " << flag << ", height: " << height_from_header << "\n";
}
Expand Down Expand Up @@ -336,7 +345,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
<< site_name_col << "\" from the input \"" << ascii_file.filename()
<< "\"\n\n";
}
else if (_stationId != data_line[sid_idx]) {
else if (_stationId != data_line[sid_idx] && _stationId != SITE_MISSING) {
mlog << Error << "\nAeronetHandler::_readObservations() The header and data columns don't match."
<< " The station ID from data column (" << data_line[sid_idx] << ") at " << sid_idx
<< " is different from " << _stationId
Expand All @@ -354,7 +363,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
if (elv_idx >= 0) _stationAlt = atof(data_line[elv_idx]);
else _stationAlt = bad_data_float;

mlog << Debug(5) << "AeronetHandler::_readObservations() stationID: "
mlog << Debug(7) << "AeronetHandler::_readObservations() stationID: "
<< ((sid_idx < 0) ? _stationId : data_line[sid_idx]) << " from index " << sid_idx
<< " lat: " << _stationLat
<< " lon: " << _stationLon
Expand All @@ -366,7 +375,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
// Pull the valid time from the data line
//

time_t valid_time = _getValidTime(data_line);
time_t valid_time = _getValidTime(data_line, (0 <= month_idx ? month_idx : date_idx));

if (valid_time == 0)
return false;
Expand All @@ -385,7 +394,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
for (int k = 0; k < process_flag.n_elements(); k++)
{
if (process_flag[k] != 1) continue;

string hdr_field = hdr_tokens[k];
size_t found_aot = hdr_field.find(aot);
//int found_angstrom = hdr_field.find(angstrom);
Expand All @@ -396,15 +404,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
height = hdr_field.substr((found_aot + 4), hdr_field.size() - 1);
}

//if (found_angstrom != string::npos)
//{
// size_t found_dash = hdr_field.find("-");
// if (found_dash != string::npos)
// {
// height = hdr_field.substr(0, found_dash);
// }
//}

if(strcmp(data_line[k], AERONET_NA_STR) == 0) continue;

var_name = AOT_NAME;
Expand All @@ -424,7 +423,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
}
}

_addObservations(Observation(header_type, _stationId,
_addObservations(Observation(header_type, (sid_idx<0 ? _stationId : data_line[sid_idx]),
valid_time,
_stationLat, _stationLon,
_stationAlt,
Expand All @@ -435,20 +434,22 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
var_name));
}
if (format_version == 3) {
if (!has_aod_at_550 && !is_eq(aod_at_440, bad_data_float) && !is_eq(aod_at_675, bad_data_float)) {
// if (!has_aod_at_550 && aod_at_440 >= 0 && aod_at_675 >= 0) {
var_id = aod_var_id;
var_name = AOD_NAME;
double dheight = 550;
double aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,dheight);
_addObservations(Observation(header_type, _stationId, valid_time,
_stationLat, _stationLon, _stationAlt,
na_str, var_id, bad_data_double, dheight,
aod_at_550,
var_name));
mlog << Debug(7) << "AeronetHandler::_readObservations() AOD at 550: "
<< aod_at_550 << "\t440: " << aod_at_440
<< "\t675: " << aod_at_675 << "\n";
}
if (!is_eq(aod_at_550, bad_data_double)) {
_addObservations(Observation(header_type, (sid_idx<0 ? _stationId : data_line[sid_idx]), valid_time,
_stationLat, _stationLon, _stationAlt,
na_str, var_id, bad_data_double, dheight,
aod_at_550,
var_name));
mlog << Debug(7) << "AeronetHandler::_readObservations() AOD at 550: "
<< aod_at_550 << "\t440: " << aod_at_440
<< "\t675: " << aod_at_675 << "\n";
}
// }
}
} // end while
if (bad_line_count > 0) {
Expand All @@ -458,15 +459,13 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
<< ascii_file.filename() << "\".\n\n";
}

if (format_version == 3) {
if (format_version == 3 && test_AOD_550) {
double aod_at_675, aod_at_440;
double aod_at_550_expected, aod_at_550;
//double angstrom_675_440_expected, angstrom_675_440;

aod_at_675 = 0.645283;
aod_at_440 = 0.794593;
aod_at_550_expected = 0.71286864;
//angstrom_675_440_expected = 0.486381371;
aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,550);
if (! is_eq(aod_at_550, aod_at_550_expected))
mlog << Warning << "AeronetHandler::_readObservations() Check AOD at 550: "
Expand All @@ -484,7 +483,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)
aod_at_675 = 0.669274;
aod_at_440 = 0.83858;
aod_at_550_expected = 0.745546058;
//angstrom_675_440_expected = 0.526983959;
aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,550);
if (! is_eq(aod_at_550, aod_at_550_expected))
mlog << Warning << "AeronetHandler::_readObservations() Check AOD at 550: "
Expand All @@ -504,29 +502,44 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file)

////////////////////////////////////////////////////////////////////////

time_t AeronetHandler::_getValidTime(const DataLine &data_line) const
time_t AeronetHandler::_getValidTime(const DataLine &data_line, int date_offset) const
{
//
// Pull out the date information
//

ConcatString date_string(data_line[0]);
string mday;
string mon;
string year;
int month = -1;
bool date_yyyymmm = false;
ConcatString date_string(data_line[date_offset]);
StringArray dateTokens = date_string.split(":");
if (1 == dateTokens.n_elements()) {
mlog << Error << "\nAeronetHandler::_getValidTime -> "
<< "Not supported date: \"" << date_string << "\".\n\n";
return 0;
// Support "yyyy-MMM"
StringArray ymTokens = date_string.split("-");
if (2 == ymTokens.n_elements()) {
year = ymTokens[0];
month = month_name_to_m(ymTokens[1].c_str());
if (0 < month && month <= 12) date_yyyymmm = true;
}
if (!date_yyyymmm) {
mlog << Error << "\nAeronetHandler::_getValidTime -> "
<< "Not supported date: \"" << date_string << "\".\n\n";
return 0;
}
}
else {
mday = dateTokens[0];
mon = dateTokens[1];
year = dateTokens[2];
}

string mday = dateTokens[0];
string mon = dateTokens[1];
string year = dateTokens[2];

//
// Pull out the time information
//

ConcatString time_string(data_line[1]);
ConcatString time_string(data_line[date_offset+1]);
StringArray timeTokens = time_string.split(":");

//
Expand All @@ -537,9 +550,9 @@ time_t AeronetHandler::_getValidTime(const DataLine &data_line) const
memset(&time_struct, 0, sizeof(time_struct));

time_struct.tm_year = atoi(year.c_str()) - 1900;
time_struct.tm_mon = atoi(mon.c_str()) - 1;
time_struct.tm_mday = atoi(mday.c_str());
if (3 <= timeTokens.n_elements()) {
time_struct.tm_mon = ((0 < month) ? month : atoi(mon.c_str())) - 1;
time_struct.tm_mday = (0 < month) ? 1 : atoi(mday.c_str());
if (3 <= timeTokens.n_elements() && !date_yyyymmm) {
string hour = timeTokens[0];
string min = timeTokens[1];
string sec = timeTokens[2];
Expand Down Expand Up @@ -573,7 +586,8 @@ bool AeronetHandler::_readHeaderInfo(LineDataFile &ascii_file)

if (format_version == 3) {
_stationId = data_line[0];
if (' ' == _stationId[0]) _stationId = _stationId.substr(1);
if (string::npos != _stationId.find(AERONET_V3_STR)) _stationId = SITE_MISSING;
else if (' ' == _stationId[0]) _stationId = _stationId.substr(1);
mlog << Debug(5) << " _stationId: [" << _stationId << "]\n";
// read lat/lon from https://aeronet.gsfc.nasa.gov/aeronet_locations_v3.txt
return true;
Expand Down Expand Up @@ -770,7 +784,10 @@ string AeronetHandler::make_var_name_from_header(string hdr_field) {

double angstrom_power_interplation(double value_1, double value_2,
double level_1, double level_2, double target_level) {
double angstrom_log = -log10(value_1/value_2)/log10(level_1/level_2);
double angstrom_value = value_2 * pow((target_level/level_2),-angstrom_log);
double angstrom_value = bad_data_double;
if ((value_1*value_2) >=0 && (level_1*level_2) >=0) {
double angstrom_log = -log10(value_1/value_2)/log10(level_1/level_2);
angstrom_value = value_2 * pow((target_level/level_2),-angstrom_log);
}
return angstrom_value;
}
2 changes: 1 addition & 1 deletion met/src/tools/other/ascii2nc/aeronet_handler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ class AeronetHandler : public FileHandler

// Get the observation valid time from the given observation line

time_t _getValidTime(const DataLine &data_line) const;
time_t _getValidTime(const DataLine &data_line, int date_offset) const;

// Read the observations from the given file and add them to the
// _observations vector.
Expand Down