Added test

KNMI · Jun 28, 2023 · 3786820 · 3786820
1 parent 118e432
commit 3786820
Show file tree

Hide file tree

Showing 9 changed files with 313 additions and 16 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,8 @@
+**Version 2.11.0 2023-06-28**
+- Support for DataPostProcessor Operator, to add, substract, multiply or divide layers
+- Dimensions can now be fixated and hidden, to make layers representing a single elevation based on a variable with multiple elevations
+- Support to substract two elevations from the same variable
+
 **Version 2.10.4 2023-06-01**
 - Fix: Racmo datasets with rotated_pole projection does work again
 

diff --git a/adagucserverEC/CDataPostProcessors/CDataPostProcessor_Operator.cpp b/adagucserverEC/CDataPostProcessors/CDataPostProcessor_Operator.cpp
@@ -70,52 +70,103 @@ int CDPPOperator::execute(CServerConfig::XMLE_DataPostProc *proc, CDataSource *d
   if (mode == CDATAPOSTPROCESSOR_RUNAFTERREADING) {
     CDBDebug("CDATAPOSTPROCESSOR_RUNAFTERREADING::Applying OPERATOR");
     size_t l = (size_t)dataSource->dHeight * (size_t)dataSource->dWidth;
-    CDF::allocateData(dataSource->getDataObject(0)->cdfVariable->getType(), &dataSource->getDataObject(0)->cdfVariable->data, l);
+    dataSource->getDataObject(0)->cdfVariable->data = NULL;
+    CDF::allocateData(CDF_FLOAT, &dataSource->getDataObject(0)->cdfVariable->data, l);
 
     float *result = (float *)dataSource->getDataObject(0)->cdfVariable->data;
 
     CDataSource::DataObject *dataObjectA, *dataObjectB;
     try {
       dataObjectA = dataSource->getDataObject(proc->attr.a);
-      dataObjectB = dataSource->getDataObject(proc->attr.b);
     } catch (int e) {
+      CDBError("Variable %s not found", proc->attr.a.c_str());
       return 1;
     }
 
-    float *a = (float *)dataObjectA->cdfVariable->data;
-    float *b = (float *)dataObjectB->cdfVariable->data;
-
-    if (proc->attr.mode.equals("substract")) {
-      for (size_t j = 0; j < l; j++) {
-        result[j] = a[j] - b[j];
-      }
+    try {
+      dataObjectB = dataSource->getDataObject(proc->attr.b);
+    } catch (int e) {
+      CDBError("Variable %s not found", proc->attr.b.c_str());
+      return 1;
     }
 
+    void *dataA = dataObjectA->cdfVariable->data;
+    void *dataB = dataObjectB->cdfVariable->data;
+    CDFType typeA = dataObjectA->cdfVariable->getType();
+    CDFType typeB = dataObjectB->cdfVariable->getType();
+
     if (proc->attr.mode.equals("-")) {
       for (size_t j = 0; j < l; j++) {
-        result[j] = a[j] - b[j];
+        float a = getElement(dataA, typeA, j);
+        float b = getElement(dataB, typeB, j);
+        result[j] = a - b;
       }
     }
     if (proc->attr.mode.equals("+")) {
       for (size_t j = 0; j < l; j++) {
-        result[j] = a[j] + b[j];
+        float a = getElement(dataA, typeA, j);
+        float b = getElement(dataB, typeB, j);
+        result[j] = a + b;
       }
     }
     if (proc->attr.mode.equals("*")) {
       for (size_t j = 0; j < l; j++) {
-        result[j] = b[j] * a[j];
+        float a = getElement(dataA, typeA, j);
+        float b = getElement(dataB, typeB, j);
+        result[j] = b * a;
       }
     }
     if (proc->attr.mode.equals("/")) {
       for (size_t j = 0; j < l; j++) {
-        if (a[j] == 0) {
+        float a = getElement(dataA, typeA, j);
+        float b = getElement(dataB, typeB, j);
+        if (a == 0) {
           result[j] = NAN;
         } else {
-          result[j] = a[j] / b[j];
+          result[j] = a / b;
         }
       }
     }
   }
-  // dataSource->eraseDataObject(1);
+
+  return 0;
+}
+
+float CDPPOperator::getElement(void *data, CDFType dataType, size_t index) {
+  switch (dataType) {
+  case CDF_CHAR:
+    return ((const char *)data)[index];
+    break;
+  case CDF_BYTE:
+    return ((const char *)data)[index];
+    break;
+  case CDF_UBYTE:
+    return ((const unsigned char *)data)[index];
+    break;
+  case CDF_SHORT:
+    return ((const short *)data)[index];
+    break;
+  case CDF_USHORT:
+    return ((const ushort *)data)[index];
+    break;
+  case CDF_INT:
+    return ((const int *)data)[index];
+    break;
+  case CDF_UINT:
+    return ((const uint *)data)[index];
+    break;
+  case CDF_INT64:
+    return ((const int64_t *)data)[index];
+    break;
+  case CDF_UINT64:
+    return ((const uint64_t *)data)[index];
+    break;
+  case CDF_FLOAT:
+    return ((const float *)data)[index];
+    break;
+  case CDF_DOUBLE:
+    return ((const double *)data)[index];
+    break;
+  }
   return 0;
 }
diff --git a/adagucserverEC/CDataPostProcessors/CDataPostProcessor_Operator.h b/adagucserverEC/CDataPostProcessors/CDataPostProcessor_Operator.h
@@ -9,6 +9,7 @@
 class CDPPOperator : public CDPPInterface {
 private:
   DEF_ERRORFUNCTION();
+  float getElement(void *data, CDFType dataType, size_t index);
 
 public:
   virtual const char *getId();

diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h
@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.10.4" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.11.0" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0

diff --git a/data/config/datasets/adaguc.tests.datapostproc.xml b/data/config/datasets/adaguc.tests.datapostproc.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Configuration>
+
+    <Layer type="database" hidden="true">
+        <Name>layer_windspeed_at_2000</Name>
+        <Title>Wind speed at 2000 meter</Title>
+        <FilePath filter=".*\.nc$" gfi_openall="true">{ADAGUC_PATH}/data/datasets/netcdf_5dims/netcdf_5dims_seq1/nc_5D_20170101000000-20170101001000.nc</FilePath>
+        <Variable orgname="data" long_name="Wind speed at 2000 meter">var_wind_speed_at_2000</Variable>
+        <Styles>auto</Styles>
+        <Dimension name="time">time</Dimension>
+        <Dimension name="member">member</Dimension>
+        <Dimension name="height" fixvalue="2000" hidden="true">elevation</Dimension>
+    </Layer>
+
+
+    <Layer type="database">
+        <Name>output</Name>
+        <Title>Wind speed at 2000 minus wind speed at 8000</Title>
+        <FilePath filter=".*\.nc$" gfi_openall="true">{ADAGUC_PATH}/data/datasets/netcdf_5dims/netcdf_5dims_seq1/nc_5D_20170101000000-20170101001000.nc</FilePath>
+        <Variable orgname="data" long_name="Wind speed at 8000 meter">var_wind_speed_at_8000</Variable>
+        <Dimension name="time">time</Dimension>
+        <Dimension name="member">member</Dimension>
+        <Dimension name="height" fixvalue="8000" hidden="true">elevation</Dimension>
+
+        <DataPostProc algorithm="include_layer" name="layer_windspeed_at_2000" mode="append"/>
+        <DataPostProc algorithm="operator" mode="-" a="var_wind_speed_at_2000" b="var_wind_speed_at_8000" name="output" units="m/s"/>
+        <Min>-1</Min>
+        <Max>1</Max>
+        <Styles>auto</Styles>
+
+    </Layer>
+
+
+</Configuration>
+
+
diff --git a/tests/AdagucTests/TestDataPostProcessor.py b/tests/AdagucTests/TestDataPostProcessor.py
@@ -0,0 +1,75 @@
+import os
+import os.path
+from io import BytesIO
+import unittest
+import shutil
+import subprocess
+import json
+from lxml import etree, objectify
+import re
+from adaguc.AdagucTestTools import AdagucTestTools
+from lxml import etree, objectify
+import datetime
+
+ADAGUC_PATH = os.environ['ADAGUC_PATH']
+
+
+class TestDataPostProcessor(unittest.TestCase):
+    testresultspath = "testresults/TestDataPostProcessor/"
+    expectedoutputsspath = "expectedoutputs/TestDataPostProcessor/"
+    env = {'ADAGUC_CONFIG': ADAGUC_PATH +
+            "/data/config/adaguc.tests.dataset.xml"}
+
+    AdagucTestTools().mkdir_p(testresultspath)
+
+
+
+    def test_DataPostProcessor_SubstractLevels_GetCapabilities(self):
+        AdagucTestTools().cleanTempDir()
+        config = ADAGUC_PATH + '/data/config/adaguc.tests.dataset.xml,' + \
+            ADAGUC_PATH + '/data/config/datasets/adaguc.tests.datapostproc.xml'
+        status, data, headers = AdagucTestTools().runADAGUCServer(
+            args=['--updatedb', '--config', config], env=self.env, isCGI=False)
+        self.assertEqual(status, 0)
+        filename = "test_DataPostProcessor_SubstractLevels_GetCapabilities.xml"
+
+        status, data, headers = AdagucTestTools().runADAGUCServer(
+            "dataset=adaguc.tests.datapostproc&SERVICE=WMS&request=getcapabilities",
+            {
+                'ADAGUC_CONFIG':
+                ADAGUC_PATH + '/data/config/adaguc.tests.dataset.xml'
+            })
+
+        AdagucTestTools().writetofile(self.testresultspath + filename,
+                                      data.getvalue())
+        self.assertEqual(status, 0)
+        self.assertTrue(AdagucTestTools().compareGetCapabilitiesXML(
+            self.testresultspath + filename,
+            self.expectedoutputsspath + filename))
+
+
+
+    def test_DataPostProcessor_SubstractLevels_GetMap(self):
+        AdagucTestTools().cleanTempDir()
+        config = ADAGUC_PATH + '/data/config/adaguc.tests.dataset.xml,' + \
+            ADAGUC_PATH + '/data/config/datasets/adaguc.tests.datapostproc.xml'
+        status, data, headers = AdagucTestTools().runADAGUCServer(
+            args=['--updatedb', '--config', config], env=self.env, isCGI=False)
+        self.assertEqual(status, 0)
+        filename = "test_DataPostProcessor_SubstractLevels_GetMap.png"
+
+        status, data, headers = AdagucTestTools().runADAGUCServer(
+            "dataset=adaguc.tests.datapostproc&SERVICE=WMS&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&LAYERS=output&WIDTH=256&CRS=EPSG:4326&HEIGHT=256&STYLES=default&FORMAT=image/png&TRANSPARENT=FALSE&showlegend=false",
+            {
+                'ADAGUC_CONFIG':
+                ADAGUC_PATH + '/data/config/adaguc.tests.dataset.xml'
+            })
+
+        AdagucTestTools().writetofile(self.testresultspath + filename,
+                                      data.getvalue())
+        self.assertEqual(status, 0)
+
+        self.assertEqual(
+            data.getvalue(),
+            AdagucTestTools().readfromfile(self.expectedoutputsspath +
+                                           filename))
diff --git a/...doutputs/TestDataPostProcessor/test_DataPostProcessor_SubstractLevels_GetCapabilities.xml b/...doutputs/TestDataPostProcessor/test_DataPostProcessor_SubstractLevels_GetCapabilities.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<WMS_Capabilities
+        version="1.3.0"
+        updateSequence="0"
+        xmlns="http://www.opengis.net/wms"
+        xmlns:xlink="http://www.w3.org/1999/xlink"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.opengis.net/wms http://schemas.opengis.net/wms/1.3.0/capabilities_1_3_0.xsd"
+
+     >
+    <Service>
+        <Name>WMS</Name>
+        <Title>adaguc.tests.datapostproc</Title>
+        <Abstract>This service demonstrates how the ADAGUC server can be used to create OGC services.</Abstract>
+        <KeywordList>
+            <Keyword>view</Keyword>
+            <Keyword>infoMapAccessService</Keyword>
+            <Keyword>ADAGUCServer version 2.10.4, of Jun 28 2023 16:54:12 </Keyword>
+        </KeywordList>
+        <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;"/>
+        <ContactInformation>
+
+        </ContactInformation>
+        <Fees>no conditions apply</Fees>
+        <AccessConstraints>None</AccessConstraints>
+        <MaxWidth>8192</MaxWidth>
+        <MaxHeight>8192</MaxHeight>
+    </Service>
+    <Capability>
+        <Request>
+            <GetCapabilities>
+                <Format>text/xml</Format>
+                <DCPType><HTTP><Get><OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;"/></Get></HTTP></DCPType>
+            </GetCapabilities>
+            <GetMap>
+                <Format>image/png</Format>
+                <Format>image/png;mode=8bit</Format>
+                <Format>image/png;mode=8bit_noalpha</Format>
+                <Format>image/png;mode=24bit</Format>
+                <Format>image/png;mode=32bit</Format>
+                <Format>image/gif</Format>
+                <Format>image/jpeg</Format>
+                <!--<Format>image/webp</Format>-->
+                <DCPType><HTTP><Get><OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;"/></Get></HTTP></DCPType>
+            </GetMap>
+            <GetFeatureInfo>
+                <Format>image/png</Format>
+                <Format>text/plain</Format>
+                <Format>text/html</Format>
+                <Format>text/xml</Format>
+                <Format>application/json</Format>
+                <DCPType><HTTP><Get><OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;"/></Get></HTTP></DCPType>
+            </GetFeatureInfo>
+        </Request>
+        <Exception>
+            <Format>XML</Format>
+            <Format>INIMAGE</Format>
+            <Format>BLANK</Format>
+        </Exception>
+        <UserDefinedSymbolization SupportSLD="1" />
+<Layer>
+<Title>WMS of  adaguc.tests.datapostproc</Title>
+<CRS>EPSG:3411</CRS>
+<CRS>EPSG:3412</CRS>
+<CRS>EPSG:3575</CRS>
+<CRS>EPSG:3857</CRS>
+<CRS>EPSG:4258</CRS>
+<CRS>EPSG:4326</CRS>
+<CRS>CRS:84</CRS>
+<CRS>EPSG:25831</CRS>
+<CRS>EPSG:25832</CRS>
+<CRS>EPSG:28992</CRS>
+<CRS>EPSG:7399</CRS>
+<CRS>EPSG:50001</CRS>
+<CRS>EPSG:54030</CRS>
+<CRS>EPSG:32661</CRS>
+<CRS>EPSG:40000</CRS>
+<CRS>EPSG:900913</CRS>
+<CRS>EPSG:4326</CRS>
+<BoundingBox CRS="EPSG:3411" minx="-399240949.703125" miny="-399607437.540235" maxx="399487065.269510" maxy="398879979.546019" />
+<BoundingBox CRS="EPSG:3412" minx="-2868126705.493384" miny="-2870740092.047709" maxx="2869855924.003487" maxy="2865475278.583282" />
+<BoundingBox CRS="EPSG:3575" minx="-12712149.336113" miny="-12731583.256617" maxx="12735505.704073" maxy="12723911.899582" />
+<BoundingBox CRS="EPSG:3857" minx="-19201993.871211" miny="-22228632.557364" maxx="19982466.888021" maxy="34805382.412607" />
+<BoundingBox CRS="EPSG:4258" minx="-90.488892" miny="-180.494446" maxx="89.511108" maxy="179.505554" />
+<BoundingBox CRS="EPSG:4326" minx="-90.488892" miny="-180.494446" maxx="89.511108" maxy="179.505554" />
+<BoundingBox CRS="CRS:84" minx="-90.488892" miny="-180.494446" maxx="89.511108" maxy="179.505554" />
+<BoundingBox CRS="EPSG:25831" minx="-15931605.907041" miny="-19720315.591565" maxx="17051580.684528" maxy="19828593.536145" />
+<BoundingBox CRS="EPSG:25832" minx="-14709787.414601" miny="-19720736.728277" maxx="17177711.901099" maxy="47405035.658741" />
+<BoundingBox CRS="EPSG:28992" minx="-352494887.244074" miny="-527708138.497395" maxx="319428390.883462" maxy="521690003.576207" />
+<BoundingBox CRS="EPSG:7399" minx="-17284159.484400" miny="-8898366.230070" maxx="17986681.325950" maxy="9020047.848074" />
+<BoundingBox CRS="EPSG:50001" minx="-2000000.000000" miny="-2000000.000000" maxx="10000000.000000" maxy="8500000.000000" />
+<BoundingBox CRS="EPSG:54030" minx="-16294244.470419" miny="-8493172.970167" maxx="16956530.805045" maxy="8610630.466538" />
+<BoundingBox CRS="EPSG:32661" minx="-407173329.856895" miny="-406795071.401436" maxx="411420024.012517" maxy="411546161.390658" />
+<BoundingBox CRS="EPSG:40000" minx="-411643188.990840" miny="-411262647.285147" maxx="411891372.245993" maxy="412018271.016759" />
+<BoundingBox CRS="EPSG:900913" minx="-19201993.871211" miny="-22228632.557364" maxx="19982466.888021" maxy="34805382.412607" />
+<BoundingBox CRS="EPSG:4326" minx="89.511108" miny="-180.494446" maxx="-90.488892" maxy="179.505554" />
+<Layer queryable="1" opaque="1" cascaded="0">
+<Name>output</Name>
+<Title>Wind speed at 2000 minus wind speed at 8000</Title>
+<EX_GeographicBoundingBox>
+  <westBoundLongitude>-180.494446</westBoundLongitude>
+  <eastBoundLongitude>179.505554</eastBoundLongitude>
+  <southBoundLatitude>-90.488892</southBoundLatitude>
+  <northBoundLatitude>89.511108</northBoundLatitude>
+</EX_GeographicBoundingBox><BoundingBox CRS="EPSG:3411" minx="-399240949.703125" miny="-399607437.540235" maxx="399487065.269510" maxy="398879979.546019" />
+<BoundingBox CRS="EPSG:3412" minx="-2868126705.493384" miny="-2870740092.047709" maxx="2869855924.003487" maxy="2865475278.583282" />
+<BoundingBox CRS="EPSG:3575" minx="-12712149.336113" miny="-12731583.256617" maxx="12735505.704073" maxy="12723911.899582" />
+<BoundingBox CRS="EPSG:3857" minx="-19201993.871211" miny="-22228632.557364" maxx="19982466.888021" maxy="34805382.412607" />
+<BoundingBox CRS="EPSG:4258" minx="-90.488892" miny="-180.494446" maxx="89.511108" maxy="179.505554" />
+<BoundingBox CRS="EPSG:4326" minx="-90.488892" miny="-180.494446" maxx="89.511108" maxy="179.505554" />
+<BoundingBox CRS="CRS:84" minx="-90.488892" miny="-180.494446" maxx="89.511108" maxy="179.505554" />
+<BoundingBox CRS="EPSG:25831" minx="-15931605.907041" miny="-19720315.591565" maxx="17051580.684528" maxy="19828593.536145" />
+<BoundingBox CRS="EPSG:25832" minx="-14709787.414601" miny="-19720736.728277" maxx="17177711.901099" maxy="47405035.658741" />
+<BoundingBox CRS="EPSG:28992" minx="-352494887.244074" miny="-527708138.497395" maxx="319428390.883462" maxy="521690003.576207" />
+<BoundingBox CRS="EPSG:7399" minx="-17284159.484400" miny="-8898366.230070" maxx="17986681.325950" maxy="9020047.848074" />
+<BoundingBox CRS="EPSG:50001" minx="-2000000.000000" miny="-2000000.000000" maxx="10000000.000000" maxy="8500000.000000" />
+<BoundingBox CRS="EPSG:54030" minx="-16294244.470419" miny="-8493172.970167" maxx="16956530.805045" maxy="8610630.466538" />
+<BoundingBox CRS="EPSG:32661" minx="-407173329.856895" miny="-406795071.401436" maxx="411420024.012517" maxy="411546161.390658" />
+<BoundingBox CRS="EPSG:40000" minx="-411643188.990840" miny="-411262647.285147" maxx="411891372.245993" maxy="412018271.016759" />
+<BoundingBox CRS="EPSG:900913" minx="-19201993.871211" miny="-22228632.557364" maxx="19982466.888021" maxy="34805382.412607" />
+<BoundingBox CRS="EPSG:4326" minx="89.511108" miny="-180.494446" maxx="-90.488892" maxy="179.505554" />
+<Dimension name="time" units="ISO8601" default="2017-01-01T00:10:00Z" multipleValues="1" nearestValue="0" current="1">2017-01-01T00:00:00Z,2017-01-01T00:05:00Z,2017-01-01T00:10:00Z</Dimension>
+<Dimension name="member" units="member number" default="member1" multipleValues="1" nearestValue="0" >member6,member5,member4,member3,member2,member1</Dimension>
+   <Style>    <Name>auto/nearest</Name>    <Title>auto/nearest</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=output&amp;format=image/png&amp;STYLE=auto/nearest"/>    </LegendURL>  </Style>   <Style>    <Name>auto/bilinear</Name>    <Title>auto/bilinear</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=output&amp;format=image/png&amp;STYLE=auto/bilinear"/>    </LegendURL>  </Style>   <Style>    <Name>auto/point</Name>    <Title>auto/point</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=output&amp;format=image/png&amp;STYLE=auto/point"/>    </LegendURL>  </Style>   <Style>    <Name>auto/barb</Name>    <Title>auto/barb</Title>    <LegendURL width="300" height="400">       <Format>image/png</Format>       <OnlineResource xlink:type="simple" xlink:href="DATASET=adaguc.tests.datapostproc&amp;SERVICE=WMS&amp;&amp;version=1.1.1&amp;service=WMS&amp;request=GetLegendGraphic&amp;layer=output&amp;format=image/png&amp;STYLE=auto/barb"/>    </LegendURL>  </Style></Layer>
+    </Layer>
+  </Capability>
+</WMS_Capabilities>
diff --git a/...outputs/TestDataPostProcessor/test_DataPostProcessor_SubstractLevels_GetMap.png b/...outputs/TestDataPostProcessor/test_DataPostProcessor_SubstractLevels_GetMap.png