Merge branch 'store-layer-metadata' of github.com:KNMI/adaguc-server …

…into get-layer-metadata

adagucserverEC/CAutoConfigure.cpp

@@ -479,9 +479,6 @@ int CAutoConfigure::getFileNameForDataSource(CDataSource *dataSource, std::strin
     bool removeRequiredDims = false;
     if (dataSource->requiredDims.size() == 0) {
       removeRequiredDims = true;
-      if (CAutoConfigure::autoConfigureDimensions(dataSource) != 0) {
-        CDBWarning("Unable to autoconfigure dims");
-      }
       if (dataSource->cfgLayer->Dimension.size() > 0) {
         try {
           CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);

adagucserverEC/CDBAdapter.h

@@ -100,6 +100,9 @@ class CDBAdapter {
   virtual int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob) = 0;
   virtual CDBStore::Store *getLayerMetadataStore(const char *datasetName) = 0;
   virtual int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) = 0;
+
+  virtual bool tryAdvisoryLock(size_t key) = 0;
+  virtual bool advisoryUnLock(size_t key) = 0;
 };
 
 #endif

adagucserverEC/CDBAdapterPostgreSQL.cpp

@@ -1173,4 +1173,51 @@ int CDBAdapterPostgreSQL::dropLayerFromLayerMetadataStore(const char *datasetNam
               "WHERE datasetname='%s' AND layername = '%s';",
               datasetName, layerName);
   return dataBaseConnection->query(query.c_str());
+}
+
+bool CDBAdapterPostgreSQL::tryAdvisoryLock(size_t key) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return false;
+  }
+  CT::string query;
+  query.print("SELECT pg_try_advisory_lock(%d) as \"result\";", key);
+  auto *store = dataBaseConnection->queryToStore(query.c_str());
+  if (store == nullptr || store->getSize() != 1) {
+    CDBError("Query failed [%s]:", dataBaseConnection->getError());
+    return false;
+  }
+  auto result = store->getRecord(0)->get("result");
+  bool succesfullylocked = result != nullptr && result->equals("t");
+  if (succesfullylocked) {
+    CDBDebug("pg_try_advisory_lock succesfullylocked");
+  } else {
+    CDBDebug("pg_try_advisory_lock NOT succesfullylocked");
+  }
+  delete store;
+  return succesfullylocked;
+}
+
+bool CDBAdapterPostgreSQL::advisoryUnLock(size_t key) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return false;
+  }
+  CT::string query;
+
+  query.print("SELECT pg_advisory_unlock(%d) as \"result\";", key);
+  auto store = dataBaseConnection->queryToStore(query.c_str());
+  if (store == nullptr || store->getSize() != 1) {
+    CDBError("Query failed [%s]:", dataBaseConnection->getError());
+    return false;
+  }
+  auto result = store->getRecord(0)->get("result");
+  bool succesfullyunlocked = result != nullptr && result->equals("t");
+  if (succesfullyunlocked) {
+    CDBDebug("pg_advisory_unlock succesfullyunlocked");
+  } else {
+    CDBWarning("pg_advisory_unlock NOT succesfullyunlocked");
+  }
+  delete store;
+  return succesfullyunlocked;
 }

adagucserverEC/CDBAdapterPostgreSQL.h

@@ -93,6 +93,8 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
   CDBStore::Store *getLayerMetadataStore(const char *datasetName);
   int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName);
+  bool tryAdvisoryLock(size_t);
+  bool advisoryUnLock(size_t);
 };
 
 #endif

adagucserverEC/CDBAdapterSQLLite.h

@@ -126,6 +126,9 @@ class CDBAdapterSQLLite : public CDBAdapter {
   int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
   CDBStore::Store *getLayerMetadataStore(const char *datasetName);
   int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName);
+
+  bool tryAdvisoryLock(size_t) { return true; };
+  bool advisoryUnLock(size_t) { return true; }
 };
 
 #endif

adagucserverEC/CDBFileScanner.cpp

@@ -958,6 +958,11 @@ int CDBFileScanner::updatedb(CDataSource *dataSource, CT::string *_tailPath, CT:
           CDBDebug("Obtained filename from layer configuration [%s]", dataSource->cfgLayer->FilePath[0]->value.c_str());
         } else {
           std::string fileName;
+          if (dataSource->requiredDims.size() == 0) {
+            if (CAutoConfigure::autoConfigureDimensions(dataSource) != 0) {
+              CDBWarning("Unable to autoconfigure dims");
+            }
+          }
           if (CAutoConfigure::getFileNameForDataSource(dataSource, fileName) != 0) {
             CDBDebug("Unable to getFileNameForDataSource");
             return 1;

adagucserverEC/CRequest.cpp

@@ -3437,11 +3437,11 @@ int CRequest::updatedb(CT::string *tailPath, CT::string *layerPathToScan, int sc
   } else {
     CDBDebug("***** Finished DB Update *****");
   }
-
+  // TODO: 2024-09-20: Probably not the right place to clear these
   CDFObjectStore::getCDFObjectStore()->clear();
   CConvertGeoJSON::clearFeatureStore();
   CDFStore::clear();
-  CDBFactory::clear();
+
   return errorHasOccured > 0;
 }
 

adagucserverEC/Definitions.h

@@ -188,3 +188,5 @@
 // Client errors
 #define HTTP_STATUSCODE_404_NOT_FOUND 32
 #define HTTP_STATUSCODE_422_UNPROCESSABLE_ENTITY 33
+
+#define LOCK_METADATATABLE_ID 1

adagucserverEC/Types/LayerMetadataType.h

@@ -15,8 +15,7 @@ struct LayerMetadataDim {
 };
 
 struct LayerMetadataProjection {
-  LayerMetadataProjection() {}
-  LayerMetadataProjection(CT::string name, double bbox[]) {
+  LayerMetadataProjection(const CT::string& name, const double bbox[]) {
     this->name = name;
     for (size_t j = 0; j < 4; j++) {
       this->dfBBOX[j] = bbox[j];

adagucserverEC/adagucserver.cpp

@@ -34,6 +34,8 @@
 #include "Types/ProjectionStore.h"
 #include "utils/UpdateLayerMetadata.h"
 #include "utils/ConfigurationUtils.h"
+#include "CDBFactory.h"
+#include "CConvertGeoJSON.h"
 
 DEF_ERRORMAIN();
 
@@ -326,7 +328,19 @@ int _main(int argc, char **argv, char **) {
       CDBError("setCRequestConfigFromEnvironment failed");
       return 1;
     }
+    auto db = CDBFactory::getDBAdapter(baseRequest.getServerParams()->cfg);
+    if (db == nullptr || db->tryAdvisoryLock(LOCK_METADATATABLE_ID) == false) {
+      CDBDebug("UPDATELAYERMETADATA: Skipping updateLayerMetadata already busy (tryAdvisoryLock)");
+      return 1;
+    } else {
+      CDBDebug("UPDATELAYERMETADATA: tryAdvisoryLock obtained");
+    }
+
+    CDBDebug("UPDATELAYERMETADATA: STARTING");
     status = updateLayerMetadata(baseRequest);
+    CDBDebug("UPDATELAYERMETADATA: DONE");
+    db->advisoryUnLock(LOCK_METADATATABLE_ID);
+    CDBDebug("UPDATELAYERMETADATA: Unlocked");
     return status;
   }
 
@@ -434,6 +448,8 @@ int main(int argc, char **argv, char **envp) {
 
   CDFObjectStore::getCDFObjectStore()->clear();
 
+  CDBFactory::clear();
+
   proj_clear_cache();
   BBOXProjectionClearCache();
 

adagucserverEC/utils/ConfigurationUtils.cpp

@@ -19,3 +19,29 @@ bool checkIfPathIsFile(CT::string filePath) {
   return (filePath.endsWith(".nc") || filePath.endsWith(".h5") || filePath.endsWith(".hdf5") || filePath.endsWith(".he5") || filePath.endsWith(".png") || filePath.endsWith(".csv") ||
           filePath.endsWith(".geojson") || filePath.endsWith(".json") || filePath.startsWith("http://") || filePath.startsWith("https://") || filePath.startsWith("dodsc://"));
 }
+
+void serverLogFunctionNothing(const char *) {}
+
+/* Set config file from environment variable ADAGUC_CONFIG */
+int setCRequestConfigFromEnvironment(CRequest *request, const char *additionalDataset) {
+  char *configfile = getenv("ADAGUC_CONFIG");
+  if (configfile != NULL) {
+    CT::string configWithAdditionalDataset = configfile;
+    if (additionalDataset != nullptr && strlen(additionalDataset) > 0) {
+      configWithAdditionalDataset.concat(",");
+      configWithAdditionalDataset.concat(additionalDataset);
+    }
+    int status = request->setConfigFile(configWithAdditionalDataset.c_str());
+
+    /* Check logging level */
+    if (request->getServerParams()->isDebugLoggingEnabled() == false) {
+      setDebugFunction(serverLogFunctionNothing);
+    }
+
+    return status;
+  } else {
+    CDBError("No configuration file is set. Please set ADAGUC_CONFIG environment variable accordingly.");
+    return 1;
+  }
+  return 0;
+}

adagucserverEC/utils/ConfigurationUtils.h

@@ -4,9 +4,12 @@
 #include <vector>
 #include <string>
 #include <CServerParams.h>
+#include <CRequest.h>
 
 std::vector<std::string> getEnabledDatasetsConfigurations(CServerParams *srvParam);
 
 bool checkIfPathIsFile(CT::string filePath);
 
+int setCRequestConfigFromEnvironment(CRequest *request, const char *additionalDataset = nullptr);
+
 #endif

adagucserverEC/utils/LayerMetadataStore.cpp

@@ -249,15 +249,17 @@ int loadLayerProjectionAndExtentListFromMetadataDb(MetadataLayer *metadataLayer)
       return 1;
     }
     json a;
-    auto c = a.parse(projInfo.c_str());
-    for (auto d : c.items()) {
+    auto c = json::parse(projInfo.c_str());
+    for (const auto &d : c.items()) {
       auto bboxArray = d.value();
-      LayerMetadataProjection projection;
+      double bbox[4] = {
+          bboxArray[0].get_to((bbox[0])),
+          bboxArray[1].get_to((bbox[1])),
+          bboxArray[2].get_to((bbox[2])),
+          bboxArray[3].get_to((bbox[3])),
+      };
+      LayerMetadataProjection projection(d.key().c_str(), bbox);
       projection.name = d.key().c_str();
-      bboxArray[0].get_to((projection.dfBBOX[0]));
-      bboxArray[1].get_to((projection.dfBBOX[1]));
-      bboxArray[2].get_to((projection.dfBBOX[2]));
-      bboxArray[3].get_to((projection.dfBBOX[3]));
       metadataLayer->layerMetadata.projectionList.push_back(projection);
     }
   } catch (json::exception &e) {

adagucserverEC/utils/LayerMetadataToJson.cpp

@@ -30,7 +30,7 @@ int getLayerMetadataAsJson(CServerParams *srvParams, json &result) {
     CT::string *datasetName = record->get("datasetname");
     CT::string *layerName = record->get("layername");
     if (datasetName != nullptr && layerName != nullptr) {
-      datasetNames[record->get("datasetname")->c_str()].insert(record->get("layername")->c_str());
+      datasetNames[datasetName->c_str()].insert(layerName->c_str());
     }
   }
 

adagucserverEC/utils/UpdateLayerMetadata.cpp

@@ -7,40 +7,15 @@
 #include <json_adaguc.h>
 #include "LayerUtils.h"
 
-void serverLogFunctionNothing(const char *) {}
-
-/* Set config file from environment variable ADAGUC_CONFIG */
-int setCRequestConfigFromEnvironment(CRequest *request, const char *additionalDataset) {
-  char *configfile = getenv("ADAGUC_CONFIG");
-  if (configfile != NULL) {
-    CT::string configWithAdditionalDataset = configfile;
-    if (additionalDataset != nullptr && strlen(additionalDataset) > 0) {
-      configWithAdditionalDataset.concat(",");
-      configWithAdditionalDataset.concat(additionalDataset);
-    }
-    int status = request->setConfigFile(configWithAdditionalDataset.c_str());
-
-    /* Check logging level */
-    if (request->getServerParams()->isDebugLoggingEnabled() == false) {
-      setDebugFunction(serverLogFunctionNothing);
-    }
-
-    return status;
-  } else {
-    CDBError("No configuration file is set. Please set ADAGUC_CONFIG environment variable accordingly.");
-    return 1;
-  }
-  return 0;
-}
+typedef std::pair<std::string, std::string> DatasetAndLayerPair;
 
 int updateLayerMetadata(CRequest &request) {
   CServerParams *srvParam = request.getServerParams();
 
   auto datasetList = getEnabledDatasetsConfigurations(srvParam);
-  // TODO: Remove datasets and layers in metadatable which don't have a matching configuration
   // TODO: Remove dimension tables which don't have a matching configuration
 
-  std::map<std::string, std::set<std::string>> dataSetConfigsWithLayers;
+  std::set<DatasetAndLayerPair> dataSetConfigsWithLayers;
 
   for (auto &dataset : datasetList) {
     CT::string datasetAsCTString = dataset.c_str();
@@ -52,8 +27,7 @@ int updateLayerMetadata(CRequest &request) {
       CDBDebug("\n\n *********************************** Updating metadatatable for dataset [%s] **************************************************", dataset.c_str());
     }
     CRequest requestPerDataset;
-    // dataset = "/data/adaguc-datasets/test.uwcw_ha43_dini_5p5km_10x8.xml";
-    // dataset = "/data/adaguc-datasets/testdata.xml";
+
     int status = setCRequestConfigFromEnvironment(&requestPerDataset, dataset.c_str());
     if (status != 0) {
       CDBError("Unable to read configuration file");
@@ -67,7 +41,7 @@ int updateLayerMetadata(CRequest &request) {
         layerName.print("ID_%s", tmp.c_str());
       }
 
-      dataSetConfigsWithLayers[datasetBaseName].insert(layerName.c_str());
+      dataSetConfigsWithLayers.insert(std::make_pair(datasetBaseName, layerName.c_str()));
     }
     CT::string layerPathToScan;
     CT::string tailPath;
@@ -76,10 +50,9 @@ int updateLayerMetadata(CRequest &request) {
       CDBError("Error occured in updating the database");
       continue;
     }
-    // return 0;
   }
 
-  // Check for datasets which are not configured anymore.
+  // Check for datasets and or layers which are not configured anymore.
   json dataset;
   json layer;
 
@@ -88,43 +61,25 @@ int updateLayerMetadata(CRequest &request) {
     return 1;
   }
   auto records = layerMetaDataStore->getRecords();
-  std::map<std::string, std::set<std::string>> datasetNamesFromDB;
+
+  std::set<DatasetAndLayerPair> datasetNamesFromDB;
+
   for (auto record : records) {
     CT::string *datasetName = record->get("datasetname");
     CT::string *layerName = record->get("layername");
     if (datasetName != nullptr && layerName != nullptr) {
-      datasetNamesFromDB[record->get("datasetname")->c_str()].insert(record->get("layername")->c_str());
+      datasetNamesFromDB.insert(std::make_pair(datasetName->c_str(), layerName->c_str()));
     }
   }
 
-  std::map<std::string, std::set<std::string>> layersToDeleteFromMetadataTable;
-  for (auto datasetFromDB : datasetNamesFromDB) {
-    for (auto layerNameDb : datasetFromDB.second) {
-      bool hasLayer = false;
-      for (auto datasetConfig : dataSetConfigsWithLayers) {
-        if (datasetConfig.first == datasetFromDB.first) {
-          for (auto layerNameConfig : datasetConfig.second) {
-            if (layerNameConfig == layerNameDb) {
-              hasLayer = true;
-              break;
-            }
-          }
-        }
-      }
-      if (!hasLayer) {
-        // CDBDebug("NO: %s/%s ", datasetFromDB.first.c_str(), layerNameDb.c_str());
-        layersToDeleteFromMetadataTable[datasetFromDB.first].insert(layerNameDb);
-      } else {
-        // CDBDebug("YES: %s/%s ", datasetFromDB.first.c_str(), layerNameDb.c_str());
-      }
-    }
-  }
+  std::set<DatasetAndLayerPair> layersToDeleteFromMetadataTable;
 
-  for (auto dataset : layersToDeleteFromMetadataTable) {
-    for (auto layer : dataset.second) {
-      CDBDebug("DROP: %s/%s ", dataset.first.c_str(), layer.c_str());
-      CDBFactory::getDBAdapter(srvParam->cfg)->dropLayerFromLayerMetadataStore(dataset.first.c_str(), layer.c_str());
-    }
+  std::set_difference(datasetNamesFromDB.begin(), datasetNamesFromDB.end(), dataSetConfigsWithLayers.begin(), dataSetConfigsWithLayers.end(),
+                      std::inserter(layersToDeleteFromMetadataTable, layersToDeleteFromMetadataTable.end()));
+
+  for (auto p : layersToDeleteFromMetadataTable) {
+    CDBDebug("DROP: %s/%s ", p.first.c_str(), p.second.c_str());
+    CDBFactory::getDBAdapter(srvParam->cfg)->dropLayerFromLayerMetadataStore(p.first.c_str(), p.second.c_str());
   }
 
   return 0;

adagucserverEC/utils/UpdateLayerMetadata.h

@@ -2,8 +2,6 @@
 #define UPDATELAYERMETADATA_H
 #include <CRequest.h>
 
-int setCRequestConfigFromEnvironment(CRequest *request, const char *additionalDataset = nullptr);
-
 int updateLayerMetadata(CRequest &request);
 
 #endif