--updatelayermetadata does now clean as well

KNMI · Sep 13, 2024 · 47186df · 47186df
1 parent 552e28c
commit 47186df
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 4 deletions.
diff --git a/adagucserverEC/CAutoConfigure.cpp b/adagucserverEC/CAutoConfigure.cpp
@@ -479,6 +479,9 @@ int CAutoConfigure::getFileNameForDataSource(CDataSource *dataSource, std::strin
     bool removeRequiredDims = false;
     if (dataSource->requiredDims.size() == 0) {
       removeRequiredDims = true;
+      if (CAutoConfigure::autoConfigureDimensions(dataSource) != 0) {
+        CDBWarning("Unable to autoconfigure dims");
+      }
       if (dataSource->cfgLayer->Dimension.size() > 0) {
         try {
           CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);

diff --git a/adagucserverEC/CDBAdapter.h b/adagucserverEC/CDBAdapter.h
@@ -99,6 +99,7 @@ class CDBAdapter {
 
   virtual int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob) = 0;
   virtual CDBStore::Store *getLayerMetadataStore(const char *datasetName) = 0;
+  virtual int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) = 0;
 };
 
 #endif
diff --git a/adagucserverEC/CDBAdapterPostgreSQL.cpp b/adagucserverEC/CDBAdapterPostgreSQL.cpp
@@ -1161,3 +1161,16 @@ CDBStore::Store *CDBAdapterPostgreSQL::getLayerMetadataStore(const char *dataset
 #endif
   return layerMetaDataStore;
 }
+
+int CDBAdapterPostgreSQL::dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return -1;
+  }
+
+  CT::string query;
+  query.print("DELETE FROM layermetadata "
+              "WHERE datasetname='%s' AND layername = '%s';",
+              datasetName, layerName);
+  return dataBaseConnection->query(query.c_str());
+}
diff --git a/adagucserverEC/CDBAdapterPostgreSQL.h b/adagucserverEC/CDBAdapterPostgreSQL.h
@@ -92,6 +92,7 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   int addFilesToDataBase();
   int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
   CDBStore::Store *getLayerMetadataStore(const char *datasetName);
+  int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName);
 };
 
 #endif
diff --git a/adagucserverEC/CDBAdapterSQLLite.cpp b/adagucserverEC/CDBAdapterSQLLite.cpp
@@ -1308,5 +1308,6 @@ CDBStore::Store *CDBAdapterSQLLite::getFilesForIndices(CDataSource *dataSource,
 
 int CDBAdapterSQLLite::storeLayerMetadata(const char *, const char *, const char *, const char *) { return 0; }
 CDBStore::Store *CDBAdapterSQLLite::getLayerMetadataStore(const char *) { return nullptr; }
+int CDBAdapterSQLLite::dropLayerFromLayerMetadataStore(const char *, const char *) { return 0; };
 
 #endif
diff --git a/adagucserverEC/CDBAdapterSQLLite.h b/adagucserverEC/CDBAdapterSQLLite.h
@@ -125,6 +125,7 @@ class CDBAdapterSQLLite : public CDBAdapter {
   int addFilesToDataBase();
   int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
   CDBStore::Store *getLayerMetadataStore(const char *datasetName);
+  int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName);
 };
 
 #endif

diff --git a/adagucserverEC/CDBFileScanner.cpp b/adagucserverEC/CDBFileScanner.cpp
@@ -957,8 +957,6 @@ int CDBFileScanner::updatedb(CDataSource *dataSource, CT::string *_tailPath, CT:
           fileList.push_back(dataSource->cfgLayer->FilePath[0]->value.c_str());
           CDBDebug("Obtained filename from layer configuration [%s]", dataSource->cfgLayer->FilePath[0]->value.c_str());
         } else {
-          CDBDebug("CDBFILESCANNER_UPDATEDB_ONLYFILEFROMDEFAULTQUERY");
-
           std::string fileName;
           if (CAutoConfigure::getFileNameForDataSource(dataSource, fileName) != 0) {
             CDBDebug("Unable to getFileNameForDataSource");

diff --git a/adagucserverEC/utils/LayerMetadataStore.cpp b/adagucserverEC/utils/LayerMetadataStore.cpp
@@ -328,7 +328,6 @@ int loadLayerStyleListFromMetadataDb(MetadataLayer *metadataLayer) {
 }
 
 int storeLayerDimensionListIntoMetadataDb(MetadataLayer *metadataLayer) {
-  CDBDebug("storeLayerDimensionListIntoMetadataDb");
   try {
     json dimListJson;
     if (getDimensionListAsJson(metadataLayer, dimListJson) != 0) {

diff --git a/adagucserverEC/utils/UpdateLayerMetadata.cpp b/adagucserverEC/utils/UpdateLayerMetadata.cpp
@@ -5,6 +5,7 @@
 #include <CDBFactory.h>
 #include <CDBStore.h>
 #include <json_adaguc.h>
+#include "LayerUtils.h"
 
 void serverLogFunctionNothing(const char *) {}
 
@@ -39,9 +40,16 @@ int updateLayerMetadata(CRequest &request) {
   // TODO: Remove datasets and layers in metadatable which don't have a matching configuration
   // TODO: Remove dimension tables which don't have a matching configuration
 
+  std::map<std::string, std::set<std::string>> dataSetConfigsWithLayers;
+
   for (auto &dataset : datasetList) {
+    CT::string datasetAsCTString = dataset.c_str();
+    CT::string baseDataSetNameCT = datasetAsCTString.basename();
+    baseDataSetNameCT.replaceSelf(".xml", "");
+    std::string datasetBaseName = baseDataSetNameCT.c_str();
+
     if (dataset.length() > 0) {
-      CDBDebug("***** Scanning dataset [%s]", dataset.c_str());
+      CDBDebug("\n\n *********************************** Updating metadatatable for dataset [%s] **************************************************", dataset.c_str());
     }
     CRequest requestPerDataset;
     // dataset = "/data/adaguc-datasets/test.uwcw_ha43_dini_5p5km_10x8.xml";
@@ -51,6 +59,16 @@ int updateLayerMetadata(CRequest &request) {
       CDBError("Unable to read configuration file");
       continue;
     }
+    for (auto layer : requestPerDataset.getServerParams()->cfg->Layer) {
+      CT::string layerName;
+      makeUniqueLayerName(&layerName, layer);
+      if (isalpha(layerName.charAt(0)) == 0) {
+        CT::string tmp = layerName;
+        layerName.print("ID_%s", tmp.c_str());
+      }
+
+      dataSetConfigsWithLayers[datasetBaseName].insert(layerName.c_str());
+    }
     CT::string layerPathToScan;
     CT::string tailPath;
     status = requestPerDataset.updatedb(&tailPath, &layerPathToScan, CDBFILESCANNER_UPDATEDB | CDBFILESCANNER_DONTREMOVEDATAFROMDB | CDBFILESCANNER_UPDATEDB_ONLYFILEFROMDEFAULTQUERY, "");
@@ -60,5 +78,54 @@ int updateLayerMetadata(CRequest &request) {
     }
     // return 0;
   }
+
+  // Check for datasets which are not configured anymore.
+  json dataset;
+  json layer;
+
+  CDBStore::Store *layerMetaDataStore = CDBFactory::getDBAdapter(srvParam->cfg)->getLayerMetadataStore(nullptr);
+  if (layerMetaDataStore == nullptr) {
+    return 1;
+  }
+  auto records = layerMetaDataStore->getRecords();
+  std::map<std::string, std::set<std::string>> datasetNamesFromDB;
+  for (auto record : records) {
+    CT::string *datasetName = record->get("datasetname");
+    CT::string *layerName = record->get("layername");
+    if (datasetName != nullptr && layerName != nullptr) {
+      datasetNamesFromDB[record->get("datasetname")->c_str()].insert(record->get("layername")->c_str());
+    }
+  }
+
+  std::map<std::string, std::set<std::string>> layersToDeleteFromMetadataTable;
+  for (auto datasetFromDB : datasetNamesFromDB) {
+    for (auto layerNameDb : datasetFromDB.second) {
+      bool hasLayer = false;
+      for (auto datasetConfig : dataSetConfigsWithLayers) {
+        if (datasetConfig.first == datasetFromDB.first) {
+          for (auto layerNameConfig : datasetConfig.second) {
+            if (layerNameConfig == layerNameDb) {
+              hasLayer = true;
+              break;
+            }
+          }
+        }
+      }
+      if (!hasLayer) {
+        // CDBDebug("NO: %s/%s ", datasetFromDB.first.c_str(), layerNameDb.c_str());
+        layersToDeleteFromMetadataTable[datasetFromDB.first].insert(layerNameDb);
+      } else {
+        // CDBDebug("YES: %s/%s ", datasetFromDB.first.c_str(), layerNameDb.c_str());
+      }
+    }
+  }
+
+  for (auto dataset : layersToDeleteFromMetadataTable) {
+    for (auto layer : dataset.second) {
+      CDBDebug("DROP: %s/%s ", dataset.first.c_str(), layer.c_str());
+      CDBFactory::getDBAdapter(srvParam->cfg)->dropLayerFromLayerMetadataStore(dataset.first.c_str(), layer.c_str());
+    }
+  }
+
   return 0;
 }