Merge branch 'store-layer-metadata' into newsolarterminator

.github/workflows/docker-publish.yml

@@ -22,6 +22,17 @@ jobs:
  # ensure latest base image is used
  pull: true
  push: false
+ tags: adaguc-server
+ - name: Run Trivy vulnerability scanner
+ uses: aquasecurity/trivy-action@0.24.0
+ with:
+ image-ref: 'adaguc-server'
+ format: 'table'
+ exit-code: '1'
+ ignore-unfixed: true
+ vuln-type: 'os,library'
+ trivyignores: .trivyignore
+# severity: 'CRITICAL,HIGH'
  docker-build-and-publish:
  name: Build Docker image and potentially push to Docker Hub
  runs-on: ubuntu-latest

.trivyignore

@@ -0,0 +1,2 @@
+# Ignore secrets
+private-key

CCDFDataModel/CCDFVariable.h

@@ -292,20 +292,20 @@ namespace CDF {
  size_t getSize() { return currentSize; }
 
  Attribute *getAttribute(const char *name) const {
+ Attribute *a = getAttributeNE(name);
+ if (a == nullptr) {
+ throw(CDF_E_ATTNOTFOUND);
+ }
+ return a;
+ }
+
+ Attribute *getAttributeNE(const char *name) const {
  for (size_t j = 0; j < attributes.size(); j++) {
  if (attributes[j]->name.equals(name)) {
  return attributes[j];
  }
  }
- throw(CDF_E_ATTNOTFOUND);
- return NULL;
- }
- Attribute *getAttributeNE(const char *name) const {
- try {
- return getAttribute(name);
- } catch (int e) {
- return NULL;
- }
+ return nullptr;
  }
 
  /**

CCDFDataModel/CTime.cpp

@@ -878,7 +878,7 @@ CT::string CTime::currentDateTime() {
  strftime(buffer, 80, "%Y-%m-%dT%H:%M:%S", gmtime(&curTime.tv_sec));
 
  char currentTime[100] = "";
- snprintf(currentTime, 99, "%s:%03dZ", buffer, milli);
+ snprintf(currentTime, 99, "%s.%03dZ", buffer, milli);
 
  return currentTime;
 }

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="adaguc@knmi.nl"
 
 # Version should be same as in Definitions.h
-LABEL version="2.27.0"
+LABEL version="2.28.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \
@@ -76,7 +76,7 @@ WORKDIR /adaguc/adaguc-server-master
 
 # Upgrade pip and install python requirements.txt
 COPY requirements.txt /adaguc/adaguc-server-master/requirements.txt
-RUN pip3 install --no-cache-dir --upgrade pip pip-tools \
+RUN pip3 install --no-cache-dir --upgrade pip pip-tools setuptools \
  && pip install --no-cache-dir -r requirements.txt
 
 # Install compiled adaguc binaries from stage one
@@ -88,7 +88,7 @@ COPY python /adaguc/adaguc-server-master/python
 # To run the tests against a postgres db, see docs/test_postgesql.md
 FROM base AS test
 
-ENV TEST_IN_CONTAINER 1
+ENV TEST_IN_CONTAINER=1
 
 COPY requirements-dev.txt /adaguc/adaguc-server-master/requirements-dev.txt
 RUN pip install --no-cache-dir -r requirements-dev.txt

NEWS.md

@@ -1,3 +1,8 @@
+**Version 2.28.0 2024-09-11**
+- Metadata for layer items like variables, projections, dimensions and styles are now stored in a database table called `layermetadata`. This can be disabled via the `enablemetadatacache` property in [Settings](doc/configuration/Settings.md).
+
+
+
 **Version 2.27.0 2024-09-02**
 - PostgreSQL query from `getFilesAndIndicesForDimensions` has been rewritten, which fixes https://github.com/KNMI/adaguc-server/issues/341.
 - Optimized existing PostgreSQL queries and reduced number of PostgreSQL queries in general. This results in better performance, the benchmark tool runs 9% faster.

adagucserverEC/CAutoConfigure.cpp

@@ -463,31 +463,13 @@ int CAutoConfigure::autoConfigureStyles(CDataSource *dataSource) {
  return 0;
 }
 
-int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
+int CAutoConfigure::getFileNameForDataSource(CDataSource *dataSource, std::string &fileName) {
 
- if (dataSource == NULL) {
- CDBError("datasource == NULL");
- return 1;
- }
- if (dataSource->cfgLayer == NULL) {
- CDBError("datasource->cfgLayer == NULL");
- return 1;
- }
- if (dataSource->getNumDataObjects() == 0) {
- CDBError("dataSource->getNumDataObjects()==0");
- return 1;
- }
- if (dataSource->getDataObject(0)->cdfVariable != NULL) {
-#ifdef CAUTOCONFIGURE_DEBUG
- CDBDebug("already loaded: dataSource->getDataObject(0)->cdfVariable!=NULL");
-#endif
- return 0;
+ CT::string foundFileName = dataSource->getFileName();
+ if (foundFileName.empty()) {
+ /* Use the file specified as header file */
+ foundFileName = dataSource->headerFileName.c_str();
  }
-
- CT::string foundFileName;
-
- /* Use the file specified as header file */
- foundFileName = dataSource->headerFileName.c_str();
  if (foundFileName.empty()) {
 
  /* Try to get a file from DB */
@@ -498,7 +480,12 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
  if (dataSource->requiredDims.size() == 0) {
  removeRequiredDims = true;
  if (dataSource->cfgLayer->Dimension.size() > 0) {
- CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
+ try {
+ CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
+ } catch (ServiceExceptionCode e) {
+ CDBDebug("Unable to fillDimValuesForDataSource");
+ return 1;
+ }
  } else {
  CDBDebug("Required dims is still zero, add none now");
  COGCDims *ogcDim = new COGCDims();
@@ -509,7 +496,7 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
  ogcDim->netCDFDimName.copy("none");
  }
  }
- CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getFilesAndIndicesForDimensions(dataSource, 1000);
+ CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getFilesAndIndicesForDimensions(dataSource, 1);
  if (store != NULL && store->getSize() > 0) {
  CT::string fileNamestr = store->getRecord(0)->get(0)->c_str();
  // CDBDebug("fileName from DB: %s", fileNamestr.c_str());
@@ -531,7 +518,36 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
  return 1;
  }
  }
+ fileName = foundFileName.c_str();
+ return 0;
+}
 
+int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
+ if (dataSource == NULL) {
+ CDBError("datasource == NULL");
+ return 1;
+ }
+ if (dataSource->cfgLayer == NULL) {
+ CDBError("datasource->cfgLayer == NULL");
+ return 1;
+ }
+ if (dataSource->getNumDataObjects() == 0) {
+ CDBError("dataSource->getNumDataObjects()==0");
+ return 1;
+ }
+ if (dataSource->getDataObject(0)->cdfVariable != NULL) {
+#ifdef CAUTOCONFIGURE_DEBUG
+ CDBDebug("already loaded: dataSource->getDataObject(0)->cdfVariable!=NULL");
+#endif
+ return 0;
+ }
+
+ std::string foundFileName;
+
+ if (getFileNameForDataSource(dataSource, foundFileName) != 0) {
+ CDBDebug("Unable to getFileNameForDataSource");
+ return 1;
+ }
  /* Open a file */
  try {
  // CDBDebug("Loading header [%s]", foundFileName.c_str());

adagucserverEC/CAutoConfigure.h

@@ -56,5 +56,13 @@ class CAutoConfigure {
 public:
  static int autoConfigureDimensions(CDataSource *dataSource);
  static int autoConfigureStyles(CDataSource *dataSource);
+
+ /**
+ * Find the default filename for a datasource from the database
+ * @param dataSource
+ * @param fileName - The filename to return
+ * @returns Zero on success.
+ */
+ static int getFileNameForDataSource(CDataSource *dataSource, std::string &fileName);
 };
 #endif

adagucserverEC/CDBAdapter.h

@@ -16,7 +16,7 @@
 class CDBAdapter {
 public:
  CDBAdapter() {}
- virtual ~CDBAdapter(){};
+ virtual ~CDBAdapter() {};
 
  class GeoOptions {
  public:
@@ -96,6 +96,13 @@ class CDBAdapter {
 
  /** First use setFile<type> as many times as you whish, second use addFilesToDataBase to make it final*/
  virtual int addFilesToDataBase() = 0;
+
+ virtual int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob) = 0;
+ virtual CDBStore::Store *getLayerMetadataStore(const char *datasetName) = 0;
+ virtual int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) = 0;
+
+ virtual bool tryAdvisoryLock(size_t key) = 0;
+ virtual bool advisoryUnLock(size_t key) = 0;
 };
 
 #endif