Merge pull request ornladios#4301 from anagainaru/derived-documentation

Documentation and fixing the API for derived variables

bindings/C/adios2/c/adios2_c_io.cpp

@@ -161,11 +161,11 @@ adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const cha
     {
         adios2::helper::CheckForNullptr(io, "for adios2_io, in call to adios2_define_variable");
         adios2::core::IO &ioCpp = *reinterpret_cast<adios2::core::IO *>(io);
-        adios2::DerivedVarType typeCpp = adios2::DerivedVarType::MetadataOnly;
+        adios2::DerivedVarType typeCpp = adios2::DerivedVarType::StatsOnly;
         switch (type)
         {
         case adios2_derived_var_type_metadata_only:
-            typeCpp = adios2::DerivedVarType::MetadataOnly;
+            typeCpp = adios2::DerivedVarType::StatsOnly;
             break;
         case adios2_derived_var_type_expression_string:
             typeCpp = adios2::DerivedVarType::ExpressionString;

bindings/CXX11/adios2/cxx11/IO.h

@@ -155,9 +155,8 @@ class IO
                                const Dims &start = Dims(), const Dims &count = Dims(),
                                const bool constantDims = false);
 #ifdef ADIOS2_HAVE_DERIVED_VARIABLE
-    VariableDerived
-    DefineDerivedVariable(const std::string &name, const std::string &expression,
-                          const DerivedVarType varType = DerivedVarType::MetadataOnly);
+    VariableDerived DefineDerivedVariable(const std::string &name, const std::string &expression,
+                                          const DerivedVarType varType = DerivedVarType::StatsOnly);
 #endif
     VariableNT DefineVariable(const DataType type, const std::string &name,
                               const Dims &shape = Dims(), const Dims &start = Dims(),

bindings/Python/py11IO.h

@@ -63,9 +63,8 @@ class IO
                             const Dims &shape, const Dims &start, const Dims &count,
                             const bool isConstantDims);
 
-    VariableDerived
-    DefineDerivedVariable(const std::string &name, const std::string &expression,
-                          const DerivedVarType varType = DerivedVarType::MetadataOnly);
+    VariableDerived DefineDerivedVariable(const std::string &name, const std::string &expression,
+                                          const DerivedVarType varType = DerivedVarType::StatsOnly);
 
     Variable InquireVariable(const std::string &name);
 

bindings/Python/py11glue.cpp

@@ -126,7 +126,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .export_values();
 
     pybind11::enum_<adios2::DerivedVarType>(m, "DerivedVarType")
-        .value("MetadataOnly", adios2::DerivedVarType::MetadataOnly)
+        .value("StatsOnly", adios2::DerivedVarType::StatsOnly)
         .value("ExpressionString", adios2::DerivedVarType::ExpressionString)
         .value("StoreData", adios2::DerivedVarType::StoreData)
         .export_values();
@@ -231,7 +231,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
                  adios2::py11::IO::DefineDerivedVariable,
              pybind11::return_value_policy::move, pybind11::arg("name"),
              pybind11::arg("expression"),
-             pybind11::arg("vartype") = adios2::DerivedVarType::MetadataOnly)
+             pybind11::arg("vartype") = adios2::DerivedVarType::StatsOnly)
 
         .def("InquireVariable", &adios2::py11::IO::InquireVariable,
              pybind11::return_value_policy::move)

docs/user_guide/source/advanced/derived_variables.rst

@@ -0,0 +1,95 @@
+#################
+Derived variables
+#################
+
+Derived quantities are obtained by mathematical transformations of primary data, and they typically allow researchers to focus on specific aspects of the simulation.
+For instance, in combustion simulations that generate velocity data, calculating the magnitude of the velocity creates a derived variable that effectively identifies areas of high interest, such as regions with intense burning.
+Applications can offload the computation of derived variables to ADIOS2.
+
+.. note::
+  Examples for defining and storing derived variables can be found in examples/hello/bpStepsWriteReadDerived folder 
+
+The API to define a derived variable on the write side requires providing a math expression over primary variables and the desired type of derived variable.
+
+The math expression defines aliases for ADIOS2 variables that will be used in the expression and math operations over the aliases (example provided below). A list of supported math operations is provided at the bottom of this page.
+
+.. code-block:: c++
+
+    enum class DerivedVarType
+    {
+        StatsOnly, // only stats are saved
+        StoreData     // data is stored in addition to metadata
+    }
+
+There are currently two types of derived variables accepted by ADIOS2, one that saves only stats about the variables (min/max for each block) and one that saves data in addition to the stats, just like a primary variable.
+
+.. code-block:: c++
+
+    auto Ux = bpOut.DefineVariable<float>("var/Ux", {Nx, Ny}, {0, 0}, {Nx, Ny});
+    auto Uy = bpOut.DefineVariable<float>("var/Uy", {Nx, Ny}, {0, 0}, {Nx, Ny});
+    bpOut.DefineDerivedVariable("derived/magnitude,
+                                "x = var/Ux \n"
+                                "y = var/Uy \n"
+                                "magnitude(x, y)",
+                                adios2::DerivedVarType::StatsOnly);
+
+Derived variables can be defined at any time and are computed (and potentially stored) during the ``EndStep`` operation.
+
+The ``bpls`` utility can identify derived variables and show the math expression when the ``--show-derived`` option.
+Derived variables that store data become primary data once the write operation is done, so they are not identified as derive variables by bpls.
+
+.. code-block:: text
+
+    $ bpls StepsWriteReadDerived.bp --show-derived -l
+  float    var/Ux          10*{60000} = 0 / 45
+  float    var/Uy          10*{60000} = 0 / 90
+  float    derived/magnitude  10*{60000} = 0 / 100.623
+    Derived variable with expression: MAGNITUDE({var/Ux},{var/Uy})
+
+.. note::
+   Derived variables are currently supported only by the BP5 engine
+
+Build ADIOS2 with support for derived variables
+--------------------------
+
+By default the derived variables are ``OFF``. Building ADIOS2 with derived variables turned on requires ``-DADIOS2_USE_Derived_Variables=ON``.
+
+#################
+Supported derived operations
+#################
+
+In the current implementation, all input variables for a derived operation need to have the same type.
+
+.. list-table:: Supported derived operations
+   :widths: 20 40 40
+   :header-rows: 1
+
+   * - Operation
+     - Input
+     - Output
+     - Expression
+   * - Addition, Subtraction, Multiplication, Division
+     - All inputs must have the same dimension. Can work on multiple variables at once.
+     - Output variables will have the same type and dimension as the input variables.
+     - a+b, add(a,b), a-b, subtract(a,b), a*b, multily(a,b), a/b, divide(a,b)
+   * - Sqrt, Power
+     - Can only be applied on single variables.
+     - Return variables of the same dimension as the input variable, but of type ``long double`` (for ``long double`` input variable) or ``double`` (for the anything else).
+     - sqrt(a), pow(a)
+   * - Magnitude
+     - All inputs must have the same dimension. Can work on multiple variables at once.
+     - Output variables will have the same type and dimension as the input variables.
+     - magnitude(a, b)
+   * - Curl3D
+     - All inputs must have the same dimension. Must receive 3 variables.
+     - Output variables will have the same type and dimension as the input variables. The shape of the variable will have an extra dimension equal to 3 (e.g. for inputs of shape (d1, d2, d3), the curl variable will have shape (d1, d2, d3, 3))
+     - curl(a, b, c)
+
+
+The math operations in the table above can be combined to create complex derived expressions that are evaluated one by one. The dimensions and types need to correspond to the requirements of each operation (like in the following example).
+
+.. code-block:: text
+
+   expression= "sqrt(curl(a,b,c)) + y"
+
+The variables corresponding to a, b and c need to have the same shape and same type (example ``<int>(d1, d2, d3)``). The curl operation will generate a variable of shape (d1, d2, d3, 3) and the sqrt will generate a double typed variable of shape (d1, d2, d3, 3). For the add operation to be applied, the y variable needs to be of type double and shape (d1, d2, d3, 3).

examples/hello/CMakeLists.txt

@@ -68,3 +68,7 @@ if(ADIOS2_HAVE_SST)
   add_subdirectory(sstReader)
   add_subdirectory(sstWriter)
 endif()
+
+if(ADIOS2_HAVE_Derived_Variable)
+    add_subdirectory(bpStepsWriteReadDerived)
+endif()

examples/hello/bpStepsWriteReadDerived/CMakeLists.txt

@@ -0,0 +1,31 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+cmake_minimum_required(VERSION 3.12)
+project(ADIOS2HelloBPStepsWriteReadDerived)
+
+if(NOT TARGET adios2_core)
+  set(_components CXX)
+
+  find_package(MPI COMPONENTS C)
+  if(MPI_FOUND)
+    # Workaround for various MPI implementations forcing the link of C++ bindings
+    add_definitions(-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX)
+
+    list(APPEND _components MPI)
+  endif()
+
+  find_package(ADIOS2 REQUIRED COMPONENTS ${_components})
+endif()
+
+add_executable(adios2_hello_bpStepsWriteReadDerived bpStepsWriteReadDerived.cpp)
+target_link_libraries(adios2_hello_bpStepsWriteReadDerived adios2::cxx11)
+install(TARGETS adios2_hello_bpStepsWriteReadDerived RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+if(ADIOS2_HAVE_MPI)
+    add_executable(adios2_hello_bpStepsWriteReadDerived_mpi bpStepsWriteReadDerived.cpp)
+    target_link_libraries(adios2_hello_bpStepsWriteReadDerived_mpi adios2::cxx11_mpi MPI::MPI_CXX)
+    install(TARGETS adios2_hello_bpStepsWriteReadDerived_mpi RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif()

examples/hello/bpStepsWriteReadDerived/bpStepsWriteReadDerived.cpp

@@ -0,0 +1,149 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * bpStepsWriteReadDerived.cpp  Simple example of writing and reading two derived variables
+ * one that only stores stats and one that stores data
+ */
+#include <algorithm>
+#include <ios>
+#include <iostream>
+#if ADIOS2_USE_MPI
+#include <mpi.h>
+#endif
+#include <stdexcept>
+#include <vector>
+
+#include <adios2.h>
+
+void update_array(std::vector<float> &array, int val)
+{
+    std::transform(array.begin(), array.end(), array.begin(),
+                   [val](float v) -> float { return v + static_cast<float>(val); });
+}
+
+void writer(adios2::ADIOS &adios, const std::string &fname, const size_t Nx, unsigned int nSteps,
+            int rank, int size)
+{
+    std::vector<float> simData1(Nx, 0);
+    std::vector<float> simData2(Nx, 0);
+
+    adios2::IO bpIO = adios.DeclareIO("WriteIO");
+
+    const adios2::Dims shape{static_cast<size_t>(size * Nx)};
+    const adios2::Dims start{static_cast<size_t>(rank * Nx)};
+    const adios2::Dims count{Nx};
+    auto bpFloats1 = bpIO.DefineVariable<float>("bpFloats1", shape, start, count);
+    auto bpFloats2 = bpIO.DefineVariable<float>("bpFloats2", shape, start, count);
+
+    bpIO.DefineDerivedVariable("derived/magnitude",
+                               "x = bpFloats1 \n"
+                               "y = bpFloats2 \n"
+                               "magnitude(x, y)",
+                               adios2::DerivedVarType::StatsOnly);
+    bpIO.DefineDerivedVariable("derived/sqrt",
+                               "x = bpFloats1 \n"
+                               "sqrt(x)",
+                               adios2::DerivedVarType::StoreData);
+    adios2::Engine bpWriter = bpIO.Open(fname, adios2::Mode::Write);
+
+    for (unsigned int step = 0; step < nSteps; ++step)
+    {
+        bpWriter.BeginStep();
+        bpWriter.Put(bpFloats1, simData1.data());
+        bpWriter.Put(bpFloats2, simData2.data());
+        bpWriter.EndStep();
+
+        // Update values in the simulation data
+        update_array(simData1, 5);
+        update_array(simData2, 10);
+    }
+
+    bpWriter.Close();
+    std::cout << "Done writing " << nSteps << " steps" << std::endl;
+}
+
+void reader(adios2::ADIOS &adios, const std::string &fname, const size_t Nx, int rank)
+{
+    adios2::IO bpIO = adios.DeclareIO("ReadIO");
+
+    adios2::Engine bpReader = bpIO.Open(fname, adios2::Mode::Read);
+
+    std::vector<float> magData(Nx, 0);
+    std::vector<double> sqrtData(Nx, 0);
+    unsigned int step;
+    for (step = 0; bpReader.BeginStep() == adios2::StepStatus::OK; ++step)
+    {
+        auto bpMag = bpIO.InquireVariable<float>("derived/magnitude");
+        if (bpMag)
+        {
+            const adios2::Box<adios2::Dims> sel({{Nx * rank}, {Nx}});
+            bpMag.SetSelection(sel);
+            bpReader.Get(bpMag, magData.data());
+        }
+        auto bpSqrt = bpIO.InquireVariable<double>("derived/sqrt");
+        if (bpSqrt)
+        {
+            const adios2::Box<adios2::Dims> sel({{Nx * rank}, {Nx}});
+            bpSqrt.SetSelection(sel);
+            bpReader.Get(bpSqrt, sqrtData.data());
+        }
+
+        bpReader.EndStep();
+    }
+    bpReader.Close();
+    std::cout << "Done reading " << step << " steps" << std::endl;
+}
+
+int main(int argc, char *argv[])
+{
+    int rank, size;
+
+#if ADIOS2_USE_MPI
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+#else
+    rank = 0;
+    size = 1;
+#endif
+
+    const std::string filename = "StepsWriteReadDerived.bp";
+    const unsigned int nSteps = 10;
+    const unsigned int Nx = 60000;
+    try
+    {
+#if ADIOS2_USE_MPI
+        adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+        adios2::ADIOS adios;
+#endif
+
+        writer(adios, filename, Nx, nSteps, rank, size);
+        reader(adios, filename, Nx, rank);
+    }
+    catch (std::invalid_argument &e)
+    {
+        std::cout << "Invalid argument exception, STOPPING PROGRAM from rank " << rank << "\n";
+        std::cout << e.what() << "\n";
+    }
+    catch (std::ios_base::failure &e)
+    {
+        std::cout << "IO System base failure exception, STOPPING PROGRAM "
+                     "from rank "
+                  << rank << "\n";
+        std::cout << e.what() << "\n";
+    }
+    catch (std::exception &e)
+    {
+        std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n";
+        std::cout << e.what() << "\n";
+    }
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return 0;
+}

python/adios2/io.py

@@ -255,7 +255,7 @@ def define_derived_variable(self, name, expression, etype=None):
                 expression string using other variable names, operators and functions
 
             type
-                DerivedVarType.MetadataOnly     : store only the metadata of the derived variable
+                DerivedVarType.StatsOnly     : store only the metadata of the derived variable
                 DerivedVarType.ExpressionString : store only the definition, nothing else
                 DerivedVarType.StoreData        : store as a complete variable (data and metadata)
         """

source/adios2/common/ADIOSTypes.cpp

@@ -40,8 +40,8 @@ std::string ToString(DerivedVarType value)
 {
     switch (value)
     {
-    case DerivedVarType::MetadataOnly:
-        return "DerivedVarType::MetadataOnly";
+    case DerivedVarType::StatsOnly:
+        return "DerivedVarType::StatsOnly";
     case DerivedVarType::ExpressionString:
         return "DerivedVarType::ExpressionString";
     case DerivedVarType::StoreData:

source/adios2/common/ADIOSTypes.h

@@ -36,7 +36,7 @@ namespace adios2
 /** Type of derived variables */
 enum class DerivedVarType
 {
-    MetadataOnly,     ///< Store only the metadata (default)
+    StatsOnly,        ///< Store only the metadata (default)
     ExpressionString, ///< Store only the expression string
     StoreData         ///< Store data and metadata
 };

source/adios2/core/IO.h

@@ -185,7 +185,7 @@ class IO
 #ifdef ADIOS2_HAVE_DERIVED_VARIABLE
     VariableDerived &
     DefineDerivedVariable(const std::string &name, const std::string &expression,
-                          const DerivedVarType varType = DerivedVarType::MetadataOnly);
+                          const DerivedVarType varType = DerivedVarType::StatsOnly);
 #endif
     VariableStruct &DefineStructVariable(const std::string &name, StructDefinition &def,
                                          const Dims &shape = Dims(), const Dims &start = Dims(),

source/adios2/toolkit/format/bp5/BP5Deserializer.cpp

@@ -930,7 +930,7 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, size
                         // define the derived copy first, so we don't throw an error on actual var
                         // definition
                         auto tmp = m_Engine->m_IO.DefineDerivedVariable(
-                            VarRec->VarName, VarRec->ExprStr, DerivedVarType::MetadataOnly);
+                            VarRec->VarName, VarRec->ExprStr, DerivedVarType::StatsOnly);
                         VarRec->DerivedVariable = (void *)&tmp;
 #endif
                     }

testing/adios2/derived/TestBPDerivedCorrectness.cpp

@@ -400,7 +400,7 @@ TEST_P(DerivedCorrectnessP, CurlCorrectnessTest)
 }
 
 INSTANTIATE_TEST_SUITE_P(DerivedCorrectness, DerivedCorrectnessP,
-                         ::testing::Values(adios2::DerivedVarType::MetadataOnly,
+                         ::testing::Values(adios2::DerivedVarType::StatsOnly,
                                            adios2::DerivedVarType::StoreData));
 
 int main(int argc, char **argv)