fixed some bugs introduced recently with runners for cshelph and medi…

…anfilter; added abilty to have different asset for atl09
SlideRuleEarth · Jun 17, 2024 · 95a4301 · 95a4301
1 parent 0a3f34a
commit 95a4301
Show file tree

Hide file tree

Showing 9 changed files with 45 additions and 24 deletions.
diff --git a/clients/python/utils/bathy_runner.py b/clients/python/utils/bathy_runner.py
@@ -21,6 +21,8 @@
 parser.add_argument('--output_format',              type=str,               default="hdf5")
 parser.add_argument('--verbose',            '-v',   action='store_true',    default=False)
 parser.add_argument('--cleanup',                    action='store_true',    default=False)
+parser.add_argument('--asset',                      type=str,               default="icesat2")
+parser.add_argument('--asset09',                    type=str,               default="icesat2")
 parser.add_argument('--generate_ndwi',              action='store_true',    default=False)
 parser.add_argument('--ignore_bathy_mask',          action='store_true',    default=False)
 parser.add_argument('--print_metadata',             action='store_true',    default=False) # only available if [geo]parquet file format chosen
@@ -57,6 +59,8 @@
 
 # Set Parameters
 parms = { 
+    "asset": args.asset,
+    "atl09_asset": args.asset09,
     "srt": icesat2.SRT_DYNAMIC,
     "cnf": "atl03_not_considered",
     "pass_invalid": True,
@@ -66,9 +70,6 @@
     "spots": args.spots,
     "output_as_sdp": args.output_as_sdp,
     "classifiers": args.classifiers,
-    "surfacefinder": {
-
-    },
     "openoceans": {
         "use_ndwi": args.generate_ndwi
     },

diff --git a/packages/cre/ContainerRunner.cpp b/packages/cre/ContainerRunner.cpp
@@ -355,7 +355,7 @@ void* ContainerRunner::controlThread (void* parm)
             const long wait_http_code = CurlLib::request(EndpointObject::POST, wait_url.c_str(), NULL, &wait_response, NULL, false, false, WAIT_TIMEOUT, NULL, unix_socket);
             if(wait_http_code == EndpointObject::OK)
             {
-                alert(INFO, RTE_INFO, cr->outQ, NULL, "Container <%s> completed", cr->parms->image);
+                alert(INFO, RTE_INFO, cr->outQ, NULL, "Container <%s> completed", container_name_str.c_str());
                 done = true;
             }
             else if(wait_http_code != EndpointObject::Service_Unavailable) // curl timed out which is normal if container is still running
@@ -427,7 +427,7 @@ void* ContainerRunner::controlThread (void* parm)
             const char* stop_response = NULL;
             const long stop_http_code = CurlLib::request(EndpointObject::POST, stop_url.c_str(), NULL, &stop_response, NULL, false, false, CurlLib::DATA_TIMEOUT, NULL, unix_socket);
             if(stop_http_code != EndpointObject::No_Content) alert(CRITICAL, RTE_ERROR, cr->outQ, NULL, "Failed to force stop container <%s>: %ld - %s", cr->parms->image, stop_http_code, stop_response);
-            else alert(INFO, RTE_INFO, cr->outQ, NULL, "Force stopped container <%s> with Id %s", cr->parms->image, container_id);
+            else alert(INFO, RTE_INFO, cr->outQ, NULL, "Force stopped container <%s>", container_name_str.c_str());
             delete [] stop_response;
         }
 
@@ -436,7 +436,7 @@ void* ContainerRunner::controlThread (void* parm)
         const char* remove_response = NULL;
         const long remove_http_code = CurlLib::request(EndpointObject::DELETE, remove_url.c_str(), NULL, &remove_response, NULL, false, false, CurlLib::DATA_TIMEOUT, NULL, unix_socket);
         if(remove_http_code != EndpointObject::No_Content) alert(CRITICAL, RTE_ERROR, cr->outQ, NULL, "Failed to delete container <%s>: %ld - %s", cr->parms->image, remove_http_code, remove_response);
-        else alert(INFO, RTE_INFO, cr->outQ, NULL, "Removed container <%s> with Id %s", cr->parms->image, container_id);
+        else alert(INFO, RTE_INFO, cr->outQ, NULL, "Removed container <%s>", container_name_str.c_str());
         delete [] remove_response;
     }
 

diff --git a/plugins/icesat2/containers/oceaneyes/cshelph/runner.py b/plugins/icesat2/containers/oceaneyes/cshelph/runner.py
@@ -106,7 +106,7 @@
                                         bathymetry_label=40)
 
 df = pd.DataFrame()
-df["index_ph"] = results['index_ph']
+df["index_ph"] = point_cloud['index_ph']
 df["class_ph"] = results['classification']
 
 if output_csv != None:

diff --git a/plugins/icesat2/containers/oceaneyes/medianfilter/runner.py b/plugins/icesat2/containers/oceaneyes/medianfilter/runner.py
@@ -105,7 +105,7 @@
                                                     compress_heights=compress_heights,
                                                     compress_lats=compress_lats)
 df = pd.DataFrame()
-df["index_ph"] = results['index_ph']
+df["index_ph"] = point_cloud['index_ph']
 df["class_ph"] = results['classification']
 
 if output_csv != None:

diff --git a/plugins/icesat2/endpoints/atl24g.lua b/plugins/icesat2/endpoints/atl24g.lua
@@ -115,6 +115,14 @@ parms["t1"] = original_t1
 profile["atl09_cmr"] = (time.gps() - atl09_cmr_start_time) / 1000.0
 userlog:alert(core.INFO, core.RTE_INFO, string.format("ATL09 CMR search executed in %f seconds", profile["atl09_cmr"]))
 
+-- get ATL09 asset
+local atl09_asset_name = parms["atl09_asset"] or parms["asset"] or args.default_asset
+local atl09_asset = core.getbyname(atl09_asset_name)
+if not atl09_asset then
+    userlog:alert(core.INFO, core.RTE_ERROR, string.format("invalid asset specified for ATL09: %s", atl09_asset_name))
+    return
+end
+
 -- initialize container runtime environment
 local crenv = runner.setup()
 if not crenv.host_sandbox_directory then
@@ -124,7 +132,7 @@ end
 
 -- read ICESat-2 inputs
 local bathy_parms   = icesat2.bathyparms(parms)
-local reader        = icesat2.atl03bathy(proc.asset, resource, args.result_q, bathy_parms, geo_parms, crenv.host_sandbox_directory, false)
+local reader        = icesat2.atl03bathy(proc.asset, resource, args.result_q, bathy_parms, geo_parms, crenv.host_sandbox_directory, false, atl09_asset)
 local status        = georesource.waiton(resource, parms, nil, reader, nil, proc.sampler_disp, proc.userlog, false)
 if not status then
     userlog:alert(core.CRITICAL, core.RTE_ERROR, string.format("failed to generate ATL03 bathy inputs for %s", resource))

diff --git a/plugins/icesat2/plugin/Atl03BathyReader.cpp b/plugins/icesat2/plugin/Atl03BathyReader.cpp
@@ -114,6 +114,7 @@ const struct luaL_Reg Atl03BathyReader::LUA_META_TABLE[] = {
 int Atl03BathyReader::luaCreate (lua_State* L)
 {
     Asset* asset = NULL;
+    Asset* atl09_asset = NULL;
     BathyParms* parms = NULL;
     GeoParms* geoparms = NULL;
 
@@ -127,15 +128,17 @@ int Atl03BathyReader::luaCreate (lua_State* L)
         geoparms = dynamic_cast<GeoParms*>(getLuaObject(L, 5, GeoParms::OBJECT_TYPE, true, NULL));
         const char* shared_directory = getLuaString(L, 6);
         const bool send_terminator = getLuaBoolean(L, 7, true, true);
+        atl09_asset = dynamic_cast<Asset*>(getLuaObject(L, 8, Asset::OBJECT_TYPE));
 
         /* Return Reader Object */
-        return createLuaObject(L, new Atl03BathyReader(L, asset, resource, outq_name, parms, geoparms, shared_directory, send_terminator));
+        return createLuaObject(L, new Atl03BathyReader(L, asset, resource, outq_name, parms, geoparms, shared_directory, send_terminator, atl09_asset));
     }
     catch(const RunTimeException& e)
     {
         if(asset) asset->releaseLuaObject();
         if(parms) parms->releaseLuaObject();
         if(geoparms) geoparms->releaseLuaObject();
+        if(atl09_asset) atl09_asset->releaseLuaObject();
         mlog(e.level(), "Error creating Atl03BathyReader: %s", e.what());
         return returnLuaStatus(L, false);
     }
@@ -153,7 +156,9 @@ void Atl03BathyReader::init (void)
 /*----------------------------------------------------------------------------
  * Constructor
  *----------------------------------------------------------------------------*/
-Atl03BathyReader::Atl03BathyReader (lua_State* L, Asset* _asset, const char* _resource, const char* outq_name, BathyParms* _parms, GeoParms* _geoparms, const char* shared_directory, bool _send_terminator):
+Atl03BathyReader::Atl03BathyReader (lua_State* L, Asset* _asset, const char* _resource, 
+                                    const char* outq_name, BathyParms* _parms, GeoParms* _geoparms, 
+                                    const char* shared_directory, bool _send_terminator, Asset* _atl09_asset):
     LuaObject(L, OBJECT_TYPE, LUA_META_NAME, LUA_META_TABLE),
     missing09(false),
     read_timeout_ms(_parms->read_timeout * 1000),
@@ -164,6 +169,7 @@ Atl03BathyReader::Atl03BathyReader (lua_State* L, Asset* _asset, const char* _re
     assert(outq_name);
     assert(_parms);
     assert(_geoparms);
+    assert(_atl09_asset);
 
     /* Initialize Thread Count */
     threadCount = 0;
@@ -174,6 +180,7 @@ Atl03BathyReader::Atl03BathyReader (lua_State* L, Asset* _asset, const char* _re
     parms = _parms;
     geoparms = _geoparms;
     sharedDirectory = StringLib::duplicate(shared_directory);
+    asset09 = _atl09_asset;
 
     /* Set Signal Confidence Index */
     if(parms->surface_type == Icesat2Parms::SRT_DYNAMIC)
@@ -312,6 +319,7 @@ Atl03BathyReader::~Atl03BathyReader (void)
     delete [] resource;
 
     asset->releaseLuaObject();
+    asset09->releaseLuaObject();
 }
 
 /*----------------------------------------------------------------------------
@@ -554,7 +562,7 @@ Atl03BathyReader::Atl03Data::Atl03Data (info_t* info, const Region& region):
     h_ph                (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/h_ph").c_str(),                &info->reader->context, 0, region.first_photon,  region.num_photons),
     signal_conf_ph      (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/signal_conf_ph").c_str(),      &info->reader->context, info->reader->signalConfColIndex, region.first_photon,  region.num_photons),
     quality_ph          (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/quality_ph").c_str(),          &info->reader->context, 0, region.first_photon,  region.num_photons),
-    weight_ph           (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/weight_ph").c_str(),           &info->reader->context, 0, region.first_photon,  region.num_photons),
+//    weight_ph           (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/weight_ph").c_str(),           &info->reader->context, 0, region.first_photon,  region.num_photons),
     lat_ph              (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/lat_ph").c_str(),              &info->reader->context, 0, region.first_photon,  region.num_photons),
     lon_ph              (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/lon_ph").c_str(),              &info->reader->context, 0, region.first_photon,  region.num_photons),
     delta_time          (info->reader->asset, info->reader->resource, FString("%s/%s", info->prefix, "heights/delta_time").c_str(),          &info->reader->context, 0, region.first_photon,  region.num_photons),
@@ -578,7 +586,7 @@ Atl03BathyReader::Atl03Data::Atl03Data (info_t* info, const Region& region):
     h_ph.join(info->reader->read_timeout_ms, true);
     signal_conf_ph.join(info->reader->read_timeout_ms, true);
     quality_ph.join(info->reader->read_timeout_ms, true);
-    weight_ph.join(info->reader->read_timeout_ms, true);
+//    weight_ph.join(info->reader->read_timeout_ms, true);
     lat_ph.join(info->reader->read_timeout_ms, true);
     lon_ph.join(info->reader->read_timeout_ms, true);
     delta_time.join(info->reader->read_timeout_ms, true);
@@ -591,9 +599,9 @@ Atl03BathyReader::Atl03Data::Atl03Data (info_t* info, const Region& region):
  *----------------------------------------------------------------------------*/
 Atl03BathyReader::Atl09Class::Atl09Class (info_t* info):
     valid       (false),
-    met_u10m    (info->reader->missing09 ? NULL : info->reader->asset, info->reader->resource09.c_str(), FString("profile_%d/low_rate/met_u10m", info->track).c_str(), &info->reader->context09),
-    met_v10m    (info->reader->missing09 ? NULL : info->reader->asset, info->reader->resource09.c_str(), FString("profile_%d/low_rate/met_v10m", info->track).c_str(), &info->reader->context09),
-    delta_time  (info->reader->missing09 ? NULL : info->reader->asset, info->reader->resource09.c_str(), FString("profile_%d/low_rate/delta_time", info->track).c_str(), &info->reader->context09)
+    met_u10m    (info->reader->missing09 ? NULL : info->reader->asset09, info->reader->resource09.c_str(), FString("profile_%d/low_rate/met_u10m", info->track).c_str(), &info->reader->context09),
+    met_v10m    (info->reader->missing09 ? NULL : info->reader->asset09, info->reader->resource09.c_str(), FString("profile_%d/low_rate/met_v10m", info->track).c_str(), &info->reader->context09),
+    delta_time  (info->reader->missing09 ? NULL : info->reader->asset09, info->reader->resource09.c_str(), FString("profile_%d/low_rate/delta_time", info->track).c_str(), &info->reader->context09)
 {
     try
     {
@@ -907,11 +915,12 @@ void* Atl03BathyReader::subsettingThread (void* parm)
                 }
 
                 /* Set and Check YAPC Score */
-                const uint8_t yapc_score = atl03.weight_ph[current_photon];
-                if(yapc_score < parms->yapc.score)
-                {
-                    break;
-                }
+                const uint8_t yapc_score = 0;
+//                const uint8_t yapc_score = atl03.weight_ph[current_photon];
+//                if(yapc_score < parms->yapc.score)
+//                {
+//                    break;
+//                }
 
                 /* Check Maximum DEM Delta */
                 const double dem_delta = abs(atl03.dem_h[current_segment] - atl03.h_ph[current_photon]);

diff --git a/plugins/icesat2/plugin/Atl03BathyReader.h b/plugins/icesat2/plugin/Atl03BathyReader.h
@@ -207,7 +207,7 @@ class Atl03BathyReader: public LuaObject
                 H5Array<float>      h_ph;
                 H5Array<int8_t>     signal_conf_ph;
                 H5Array<int8_t>     quality_ph;
-                H5Array<uint8_t>    weight_ph; // yapc
+//                H5Array<uint8_t>    weight_ph; // yapc
                 H5Array<double>     lat_ph;
                 H5Array<double>     lon_ph;
                 H5Array<double>     delta_time;
@@ -294,6 +294,7 @@ class Atl03BathyReader: public LuaObject
         int                 threadCount;
         int                 numComplete;
         Asset*              asset;
+        Asset*              asset09;
         const char*         resource;
         string              resource09;
         bool                missing09;
@@ -326,7 +327,8 @@ class Atl03BathyReader: public LuaObject
                                                          BathyParms* _parms,
                                                          GeoParms* _geoparms,
                                                          const char* shared_directory,
-                                                         bool _send_terminator=true);
+                                                         bool _send_terminator,
+                                                         Asset* _atl09_asset);
                             ~Atl03BathyReader           (void) override;
 
         static void*        subsettingThread            (void* parm);

diff --git a/scripts/extensions/earth_data_query.lua b/scripts/extensions/earth_data_query.lua
@@ -54,6 +54,7 @@ ASSETS_TO_DATASETS = {
     ["icesat2-atl09"] = "ATL09",
     ["icesat2-atl13"] = "ATL13",
     ["atlas-local"] = "ATL03",
+    ["atlas-s3"] = "ATL03",
     ["nsidc-s3"] = "ATL03"
 }
 

diff --git a/targets/slideruleearth-aws/Makefile b/targets/slideruleearth-aws/Makefile
@@ -426,7 +426,7 @@ perfsubsetest: ## run subset performance test
 	$(SERVER_STAGE_DIR)/bin/sliderule $(SUBSET_PERFORMANCE_TEST)
 
 oceaneyes-test: ## run the oceaneyes container
-	docker run -it --rm -v /data:/data --name $(OCEANEYES_CONTAINER_NAME) $(ECR)/$(OCEANEYES_CONTAINER_NAME):$(VERSION) #/env/bin/python /$(ATL24_ALGORITHM)/runner.py $(ATL24_TEST_DATA_DIR)/bathy_spot_3.json $(ATL24_TEST_DATA_DIR)/bathy_spot_3.csv
+	docker run -it --rm -v /data:/data --name $(OCEANEYES_CONTAINER_NAME) $(ECR)/$(OCEANEYES_CONTAINER_NAME):$(VERSION) /env/bin/python /$(ATL24_ALGORITHM)/runner.py $(ATL24_TEST_DATA_DIR)/bathy_spot_3.json $(ATL24_TEST_DATA_DIR)/bathy_spot_3.csv
 
 coastnet-test: ## run the coastnet bathy docker container
 	docker run -it --rm -v /data:/data --name $(OCEANEYES_CONTAINER_NAME) $(ECR)/$(OCEANEYES_CONTAINER_NAME):$(VERSION) bash /coastnet/bathy.sh nil nil $(ATL24_TEST_DATA_DIR)/bathy_spot_3.csv $(ATL24_TEST_DATA_DIR)/coastnet_3.csv