Using already existing data base filename loader

KNMI · Sep 13, 2024 · 552e28c · 552e28c
1 parent a48c9d2
commit 552e28c
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 40 deletions.
diff --git a/adagucserverEC/CAutoConfigure.cpp b/adagucserverEC/CAutoConfigure.cpp
@@ -463,26 +463,7 @@ int CAutoConfigure::autoConfigureStyles(CDataSource *dataSource) {
   return 0;
 }
 
-int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
-
-  if (dataSource == NULL) {
-    CDBError("datasource == NULL");
-    return 1;
-  }
-  if (dataSource->cfgLayer == NULL) {
-    CDBError("datasource->cfgLayer == NULL");
-    return 1;
-  }
-  if (dataSource->getNumDataObjects() == 0) {
-    CDBError("dataSource->getNumDataObjects()==0");
-    return 1;
-  }
-  if (dataSource->getDataObject(0)->cdfVariable != NULL) {
-#ifdef CAUTOCONFIGURE_DEBUG
-    CDBDebug("already loaded: dataSource->getDataObject(0)->cdfVariable!=NULL");
-#endif
-    return 0;
-  }
+int CAutoConfigure::getFileNameForDataSource(CDataSource *dataSource, std::string &fileName) {
 
   CT::string foundFileName = dataSource->getFileName();
   if (foundFileName.empty()) {
@@ -499,7 +480,12 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
     if (dataSource->requiredDims.size() == 0) {
       removeRequiredDims = true;
       if (dataSource->cfgLayer->Dimension.size() > 0) {
-        CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
+        try {
+          CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
+        } catch (ServiceExceptionCode e) {
+          CDBDebug("Unable to fillDimValuesForDataSource");
+          return 1;
+        }
       } else {
         CDBDebug("Required dims is still zero, add none now");
         COGCDims *ogcDim = new COGCDims();
@@ -510,7 +496,7 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
         ogcDim->netCDFDimName.copy("none");
       }
     }
-    CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getFilesAndIndicesForDimensions(dataSource, 1000);
+    CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getFilesAndIndicesForDimensions(dataSource, 1);
     if (store != NULL && store->getSize() > 0) {
       CT::string fileNamestr = store->getRecord(0)->get(0)->c_str();
       // CDBDebug("fileName from DB: %s", fileNamestr.c_str());
@@ -532,7 +518,36 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
       return 1;
     }
   }
+  fileName = foundFileName.c_str();
+  return 0;
+}
+
+int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
+  if (dataSource == NULL) {
+    CDBError("datasource == NULL");
+    return 1;
+  }
+  if (dataSource->cfgLayer == NULL) {
+    CDBError("datasource->cfgLayer == NULL");
+    return 1;
+  }
+  if (dataSource->getNumDataObjects() == 0) {
+    CDBError("dataSource->getNumDataObjects()==0");
+    return 1;
+  }
+  if (dataSource->getDataObject(0)->cdfVariable != NULL) {
+#ifdef CAUTOCONFIGURE_DEBUG
+    CDBDebug("already loaded: dataSource->getDataObject(0)->cdfVariable!=NULL");
+#endif
+    return 0;
+  }
+
+  std::string foundFileName;
 
+  if (getFileNameForDataSource(dataSource, foundFileName) != 0) {
+    CDBDebug("Unable to getFileNameForDataSource");
+    return 1;
+  }
   /* Open a file */
   try {
     // CDBDebug("Loading header [%s]", foundFileName.c_str());

diff --git a/adagucserverEC/CAutoConfigure.h b/adagucserverEC/CAutoConfigure.h
@@ -56,5 +56,13 @@ class CAutoConfigure {
 public:
   static int autoConfigureDimensions(CDataSource *dataSource);
   static int autoConfigureStyles(CDataSource *dataSource);
+
+  /**
+   * Find the default filename for a datasource from the database
+   * @param dataSource
+   * @param fileName - The filename to return
+   * @returns Zero on success.
+   */
+  static int getFileNameForDataSource(CDataSource *dataSource, std::string &fileName);
 };
 #endif
diff --git a/adagucserverEC/CDBFileScanner.cpp b/adagucserverEC/CDBFileScanner.cpp
@@ -958,23 +958,14 @@ int CDBFileScanner::updatedb(CDataSource *dataSource, CT::string *_tailPath, CT:
           CDBDebug("Obtained filename from layer configuration [%s]", dataSource->cfgLayer->FilePath[0]->value.c_str());
         } else {
           CDBDebug("CDBFILESCANNER_UPDATEDB_ONLYFILEFROMDEFAULTQUERY");
-          int status = CRequest::setDimValuesForDataSource(dataSource, dataSource->srvParams);
-          if (status != 0) {
-            CDBError("setDimValuesForDataSource");
-            return 1;
-          }
-          status = CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
-          if (status != 0) {
-            CDBError("fillDimValuesForDataSource");
-            return 1;
-          }
-          status = CRequest::queryDimValuesForDataSource(dataSource, dataSource->srvParams);
-          if (status != 0) {
-            CDBError("queryDimValuesForDataSource");
+
+          std::string fileName;
+          if (CAutoConfigure::getFileNameForDataSource(dataSource, fileName) != 0) {
+            CDBDebug("Unable to getFileNameForDataSource");
             return 1;
           }
-          fileList.push_back(dataSource->getFileName());
-          CDBDebug("Queried file from database with filename [%s]", dataSource->getFileName());
+          fileList.push_back(fileName);
+          CDBDebug("Queried file from database with filename [%s]", fileName.c_str());
         }
       } else {
         fileList = searchFileNames(dataSource->cfgLayer->FilePath[0]->value.c_str(), filter.c_str(), tailPath.c_str());

diff --git a/adagucserverEC/utils/UpdateLayerMetadata.cpp b/adagucserverEC/utils/UpdateLayerMetadata.cpp
@@ -2,6 +2,9 @@
 #include <CDebugger.h>
 #include <CRequest.h>
 #include "ConfigurationUtils.h"
+#include <CDBFactory.h>
+#include <CDBStore.h>
+#include <json_adaguc.h>
 
 void serverLogFunctionNothing(const char *) {}
 
@@ -33,7 +36,7 @@ int updateLayerMetadata(CRequest &request) {
   CServerParams *srvParam = request.getServerParams();
 
   auto datasetList = getEnabledDatasetsConfigurations(srvParam);
-  // TODO: Remove datasets in metadatable which don't have a matching configuration
+  // TODO: Remove datasets and layers in metadatable which don't have a matching configuration
   // TODO: Remove dimension tables which don't have a matching configuration
 
   for (auto &dataset : datasetList) {
@@ -46,11 +49,14 @@ int updateLayerMetadata(CRequest &request) {
     int status = setCRequestConfigFromEnvironment(&requestPerDataset, dataset.c_str());
     if (status != 0) {
       CDBError("Unable to read configuration file");
-      return 1;
+      continue;
     }
-    status = requestPerDataset.updatedb(nullptr, nullptr, CDBFILESCANNER_UPDATEDB | CDBFILESCANNER_DONTREMOVEDATAFROMDB | CDBFILESCANNER_UPDATEDB_ONLYFILEFROMDEFAULTQUERY, "");
+    CT::string layerPathToScan;
+    CT::string tailPath;
+    status = requestPerDataset.updatedb(&tailPath, &layerPathToScan, CDBFILESCANNER_UPDATEDB | CDBFILESCANNER_DONTREMOVEDATAFROMDB | CDBFILESCANNER_UPDATEDB_ONLYFILEFROMDEFAULTQUERY, "");
     if (status != 0) {
       CDBError("Error occured in updating the database");
+      continue;
     }
     // return 0;
   }