Added advisory lock to lock layermetadatatable

KNMI · Sep 23, 2024 · fdad104 · fdad104
1 parent 41e7dd5
commit fdad104
Show file tree

Hide file tree

Showing 15 changed files with 181 additions and 64 deletions.
diff --git a/adagucserverEC/CDBAdapter.h b/adagucserverEC/CDBAdapter.h
@@ -100,6 +100,9 @@ class CDBAdapter {
   virtual int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob) = 0;
   virtual CDBStore::Store *getLayerMetadataStore(const char *datasetName) = 0;
   virtual int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) = 0;
+
+  virtual bool tryAdvisoryLock(size_t key) = 0;
+  virtual bool advisoryUnLock(size_t key) = 0;
 };
 
 #endif
diff --git a/adagucserverEC/CDBAdapterPostgreSQL.cpp b/adagucserverEC/CDBAdapterPostgreSQL.cpp
@@ -1173,4 +1173,51 @@ int CDBAdapterPostgreSQL::dropLayerFromLayerMetadataStore(const char *datasetNam
               "WHERE datasetname='%s' AND layername = '%s';",
               datasetName, layerName);
   return dataBaseConnection->query(query.c_str());
+}
+
+bool CDBAdapterPostgreSQL::tryAdvisoryLock(size_t key) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return false;
+  }
+  CT::string query;
+  query.print("SELECT pg_try_advisory_lock(%d) as \"result\";", key);
+  auto *store = dataBaseConnection->queryToStore(query.c_str());
+  if (store == nullptr || store->getSize() != 1) {
+    CDBError("Query failed [%s]:", dataBaseConnection->getError());
+    return false;
+  }
+  auto result = store->getRecord(0)->get("result");
+  bool succesfullylocked = result != nullptr && result->equals("t");
+  if (succesfullylocked) {
+    CDBDebug("pg_try_advisory_lock succesfullylocked");
+  } else {
+    CDBDebug("pg_try_advisory_lock NOT succesfullylocked");
+  }
+  delete store;
+  return succesfullylocked;
+}
+
+bool CDBAdapterPostgreSQL::advisoryUnLock(size_t key) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return false;
+  }
+  CT::string query;
+
+  query.print("SELECT pg_advisory_unlock(%d) as \"result\";", key);
+  auto store = dataBaseConnection->queryToStore(query.c_str());
+  if (store == nullptr || store->getSize() != 1) {
+    CDBError("Query failed [%s]:", dataBaseConnection->getError());
+    return false;
+  }
+  auto result = store->getRecord(0)->get("result");
+  bool succesfullyunlocked = result != nullptr && result->equals("t");
+  if (succesfullyunlocked) {
+    CDBDebug("pg_advisory_unlock succesfullyunlocked");
+  } else {
+    CDBWarning("pg_advisory_unlock NOT succesfullyunlocked");
+  }
+  delete store;
+  return succesfullyunlocked;
 }
diff --git a/adagucserverEC/CDBAdapterPostgreSQL.h b/adagucserverEC/CDBAdapterPostgreSQL.h
@@ -93,6 +93,8 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
   CDBStore::Store *getLayerMetadataStore(const char *datasetName);
   int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName);
+  bool tryAdvisoryLock(size_t);
+  bool advisoryUnLock(size_t);
 };
 
 #endif
diff --git a/adagucserverEC/CDBAdapterSQLLite.h b/adagucserverEC/CDBAdapterSQLLite.h
@@ -126,6 +126,9 @@ class CDBAdapterSQLLite : public CDBAdapter {
   int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
   CDBStore::Store *getLayerMetadataStore(const char *datasetName);
   int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName);
+
+  bool tryAdvisoryLock(size_t) { return true; };
+  bool advisoryUnLock(size_t) { return true; }
 };
 
 #endif

diff --git a/adagucserverEC/CDBFileScanner.cpp b/adagucserverEC/CDBFileScanner.cpp
@@ -968,7 +968,7 @@ int CDBFileScanner::updatedb(CDataSource *dataSource, CT::string *_tailPath, CT:
             return 1;
           }
           fileList.push_back(fileName);
-          CDBDebug("Queried file from database with filename [%s]", fileName.c_str());
+          // CDBDebug("Queried file from database with filename [%s]", fileName.c_str());
         }
       } else {
         fileList = searchFileNames(dataSource->cfgLayer->FilePath[0]->value.c_str(), filter.c_str(), tailPath.c_str());

diff --git a/adagucserverEC/CRequest.cpp b/adagucserverEC/CRequest.cpp
@@ -3422,11 +3422,11 @@ int CRequest::updatedb(CT::string *tailPath, CT::string *layerPathToScan, int sc
   } else {
     CDBDebug("***** Finished DB Update *****");
   }
-
+  // TODO: 2024-09-20: Probably not the right place to clear these
   CDFObjectStore::getCDFObjectStore()->clear();
   CConvertGeoJSON::clearFeatureStore();
   CDFStore::clear();
-  CDBFactory::clear();
+
   return errorHasOccured > 0;
 }
 

diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h
@@ -188,3 +188,5 @@
 // Client errors
 #define HTTP_STATUSCODE_404_NOT_FOUND 32
 #define HTTP_STATUSCODE_422_UNPROCESSABLE_ENTITY 33
+
+#define LOCK_METADATATABLE_ID 1
diff --git a/adagucserverEC/adagucserver.cpp b/adagucserverEC/adagucserver.cpp
@@ -34,6 +34,8 @@
 #include "Types/ProjectionStore.h"
 #include "utils/UpdateLayerMetadata.h"
 #include "utils/ConfigurationUtils.h"
+#include "CDBFactory.h"
+#include "CConvertGeoJSON.h"
 
 DEF_ERRORMAIN();
 
@@ -326,7 +328,19 @@ int _main(int argc, char **argv, char **) {
       CDBError("setCRequestConfigFromEnvironment failed");
       return 1;
     }
+    auto db = CDBFactory::getDBAdapter(baseRequest.getServerParams()->cfg);
+    if (db == nullptr || db->tryAdvisoryLock(LOCK_METADATATABLE_ID) == false) {
+      CDBDebug("UPDATELAYERMETADATA: Skipping updateLayerMetadata already busy (tryAdvisoryLock)");
+      return 1;
+    } else {
+      CDBDebug("UPDATELAYERMETADATA: tryAdvisoryLock obtained");
+    }
+
+    CDBDebug("UPDATELAYERMETADATA: STARTING");
     status = updateLayerMetadata(baseRequest);
+    CDBDebug("UPDATELAYERMETADATA: DONE");
+    db->advisoryUnLock(LOCK_METADATATABLE_ID);
+    CDBDebug("UPDATELAYERMETADATA: Unlocked");
     return status;
   }
 
@@ -434,6 +448,8 @@ int main(int argc, char **argv, char **envp) {
 
   CDFObjectStore::getCDFObjectStore()->clear();
 
+  CDBFactory::clear();
+
   proj_clear_cache();
   BBOXProjectionClearCache();
 

diff --git a/adagucserverEC/utils/LayerMetadataStore.cpp b/adagucserverEC/utils/LayerMetadataStore.cpp
@@ -250,15 +250,14 @@ int loadLayerProjectionAndExtentListFromMetadataDb(MetadataLayer *metadataLayer)
     }
     json a;
     auto c = json::parse(projInfo.c_str());
-    for (const auto& d : c.items()) {
+    for (const auto &d : c.items()) {
       auto bboxArray = d.value();
-      double bbox[4] =
-          {
-              bboxArray[0].get_to((bbox[0])),
-              bboxArray[1].get_to((bbox[1])),
-              bboxArray[2].get_to((bbox[2])),
-              bboxArray[3].get_to((bbox[3])),
-          };
+      double bbox[4] = {
+          bboxArray[0].get_to((bbox[0])),
+          bboxArray[1].get_to((bbox[1])),
+          bboxArray[2].get_to((bbox[2])),
+          bboxArray[3].get_to((bbox[3])),
+      };
       LayerMetadataProjection projection(d.key().c_str(), bbox);
       projection.name = d.key().c_str();
       metadataLayer->layerMetadata.projectionList.push_back(projection);

diff --git a/adagucserverEC/utils/LayerMetadataToJson.cpp b/adagucserverEC/utils/LayerMetadataToJson.cpp
@@ -30,7 +30,7 @@ int getLayerMetadataAsJson(CServerParams *srvParams, json &result) {
     CT::string *datasetName = record->get("datasetname");
     CT::string *layerName = record->get("layername");
     if (datasetName != nullptr && layerName != nullptr) {
-      datasetNames[record->get("datasetname")->c_str()].insert(record->get("layername")->c_str());
+      datasetNames[datasetName->c_str()].insert(layerName->c_str());
     }
   }
 

diff --git a/adagucserverEC/utils/UpdateLayerMetadata.cpp b/adagucserverEC/utils/UpdateLayerMetadata.cpp
@@ -93,7 +93,7 @@ int updateLayerMetadata(CRequest &request) {
     CT::string *datasetName = record->get("datasetname");
     CT::string *layerName = record->get("layername");
     if (datasetName != nullptr && layerName != nullptr) {
-      datasetNamesFromDB[record->get("datasetname")->c_str()].insert(record->get("layername")->c_str());
+      datasetNamesFromDB[datasetName->c_str()].insert(layerName->c_str());
     }
   }
 

diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py
@@ -72,11 +72,14 @@ def get_random_string(self, length):
     def setConfiguration(self, configFile):
         self.ADAGUC_CONFIG = configFile
 
-    def scanDataset(self, datasetName):
-        config = self.ADAGUC_CONFIG + "," + datasetName
-        """ Setup a new environment """
-        adagucenv = {}
-        """ Set required environment variables """
+    def isLoggingEnabled(self):
+        return os.getenv(
+            "ADAGUC_ENABLELOGBUFFER", "TRUE"
+        ) != "DISABLELOGGING"
+
+    def getAdagucEnv(self, adagucenv = {}):
+
+        """ Set required environment variables for adaguc to run"""
         adagucenv["ADAGUC_CONFIG"] = self.ADAGUC_CONFIG
         adagucenv["ADAGUC_LOGFILE"] = self.ADAGUC_LOGFILE
         adagucenv["ADAGUC_PATH"] = self.ADAGUC_PATH
@@ -85,26 +88,46 @@ def scanDataset(self, datasetName):
         adagucenv["ADAGUC_DATASET_DIR"] = self.ADAGUC_DATASET_DIR
         adagucenv["ADAGUC_TMP"] = self.ADAGUC_TMP
         adagucenv["ADAGUC_FONT"] = self.ADAGUC_FONT
+        adagucenv["ADAGUC_DATARESTRICTION"] = "FALSE"
+        if os.getenv("ADAGUC_DB"):
+            adagucenv["ADAGUC_DB"] = os.getenv("ADAGUC_DB")
+        adagucenv["ADAGUC_ENABLELOGBUFFER"] = os.getenv(
+            "ADAGUC_ENABLELOGBUFFER", "TRUE"
+        )
+        ld_library_path = os.getenv("LD_LIBRARY_PATH")
+        if ld_library_path:
+            adagucenv["LD_LIBRARY_PATH"] = ld_library_path
+
+
+        return adagucenv
 
+    def updateLayerMetadata(self):
+        """Uses the adaguc executable to update the layermetadatatable"""
+        adagucenv = self.getAdagucEnv()
         status, data, headers = asyncio.run(
             self.runADAGUCServer(
-                args=["--updatedb", "--config", config], env=adagucenv, isCGI=False
+                args=["--updatelayermetadata"], env=adagucenv, isCGI=False, showLogOnError = False
             )
         )
 
+
+        return status, data.getvalue().decode()        
+
+    def scanDataset(self, datasetName):
+        config = self.ADAGUC_CONFIG + "," + datasetName
+        adagucenv = self.getAdagucEnv()
+        status, data, headers = asyncio.run(
+            self.runADAGUCServer(
+                args=["--updatedb", "--config", config], env=adagucenv, isCGI=False, showLogOnError = False
+            )
+        )
+
+
         return data.getvalue().decode()
 
     def runGetMapUrl(self, url):
-        adagucenv = {}
-        """ Set required environment variables """
-        adagucenv["ADAGUC_CONFIG"] = self.ADAGUC_CONFIG
-        adagucenv["ADAGUC_LOGFILE"] = self.ADAGUC_LOGFILE
-        adagucenv["ADAGUC_PATH"] = self.ADAGUC_PATH
-        adagucenv["ADAGUC_DATA_DIR"] = self.ADAGUC_DATA_DIR
-        adagucenv["ADAGUC_AUTOWMS_DIR"] = self.ADAGUC_AUTOWMS_DIR
-        adagucenv["ADAGUC_DATASET_DIR"] = self.ADAGUC_DATASET_DIR
-        adagucenv["ADAGUC_TMP"] = self.ADAGUC_TMP
-        adagucenv["ADAGUC_FONT"] = self.ADAGUC_FONT
+        adagucenv = self.getAdagucEnv()
+
         status, data, headers = asyncio.run(
             self.runADAGUCServer(url, env=adagucenv, showLogOnError=False)
         )
@@ -148,7 +171,7 @@ def cache_wanted(self, url: str):
 
         Returns: boolean
         """
-        if not runAdaguc.use_cache:
+        if not runAdaguc.use_cache or not url:
             return False
         if "request=getcapabilities" in url.lower():
             return True
@@ -164,26 +187,10 @@ async def runADAGUCServer(
         showLogOnError=True,
         showLog=False,
     ):
-        # adagucenv=os.environ.copy()
-        # adagucenv.update(env)
 
-        adagucenv = env
+        adagucenv = self.getAdagucEnv(env)
+
 
-        adagucenv["ADAGUC_ENABLELOGBUFFER"] = os.getenv(
-            "ADAGUC_ENABLELOGBUFFER", "TRUE"
-        )
-        adagucenv["ADAGUC_CONFIG"] = self.ADAGUC_CONFIG
-        adagucenv["ADAGUC_LOGFILE"] = self.ADAGUC_LOGFILE
-        adagucenv["ADAGUC_PATH"] = self.ADAGUC_PATH
-        adagucenv["ADAGUC_DATARESTRICTION"] = "FALSE"
-        adagucenv["ADAGUC_DATA_DIR"] = self.ADAGUC_DATA_DIR
-        adagucenv["ADAGUC_AUTOWMS_DIR"] = self.ADAGUC_AUTOWMS_DIR
-        adagucenv["ADAGUC_DATASET_DIR"] = self.ADAGUC_DATASET_DIR
-        adagucenv["ADAGUC_TMP"] = self.ADAGUC_TMP
-        adagucenv["ADAGUC_FONT"] = self.ADAGUC_FONT
-        ld_library_path = os.getenv("LD_LIBRARY_PATH")
-        if ld_library_path:
-            adagucenv["LD_LIBRARY_PATH"] = ld_library_path
 
         # Forward all environment variables starting with ADAGUCENV_
         prefix: str = "ADAGUCENV_"
@@ -242,7 +249,7 @@ async def runADAGUCServer(
                 print("=== START ADAGUC HTTP HEADER ===")
                 print(headers)
                 print("=== END ADAGUC HTTP HEADER ===")
-            else:
+            elif isCGI:
                 print("Process: No HTTP Headers written")
 
             print("--- END ADAGUC DEBUG INFO ---\n")