fixup! fixup! python: rewrite Python high level API in python

bindings/Python/py11IO.cpp

@@ -170,20 +170,20 @@ Attribute IO::DefineAttribute(const std::string &name, const std::vector<std::st
                                                          variableName, separator));
 }
 
-Attribute IO::InquireAttribute(const std::string &name)
+Attribute IO::InquireAttribute(const std::string &name, const std::string &variableName, const std::string separator)
 {
     helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::InquireAttribute");
 
     core::AttributeBase *attribute = nullptr;
-    const DataType type(m_IO->InquireAttributeType(name));
+    const DataType type(m_IO->InquireAttributeType(name, variableName, separator));
 
     if (type == DataType::None)
     {
     }
 #define declare_template_instantiation(T)                                                          \
     else if (type == helper::GetDataType<T>())                                                     \
     {                                                                                              \
-        attribute = m_IO->InquireAttribute<T>(name);                                               \
+        attribute = m_IO->InquireAttribute<T>(name, variableName, separator);                                               \
     }
     ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation

bindings/Python/py11IO.h

@@ -71,7 +71,7 @@ class IO
                               const std::string &variableName = "",
                               const std::string separator = "/");
 
-    Attribute InquireAttribute(const std::string &name);
+    Attribute InquireAttribute(const std::string &name, const std::string &variableName = "", const std::string separator = "/");
 
     bool RemoveVariable(const std::string &name);
 

bindings/Python/py11glue.cpp

@@ -206,8 +206,13 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .def("InquireVariable", &adios2::py11::IO::InquireVariable,
              pybind11::return_value_policy::move)
 
-        .def("InquireAttribute", &adios2::py11::IO::InquireAttribute,
-             pybind11::return_value_policy::move)
+        .def("InquireAttribute",
+             (adios2::py11::Attribute(adios2::py11::IO::*)(
+                 const std::string &, const std::string &,
+                 const std::string)) &
+                 adios2::py11::IO::InquireAttribute,
+             pybind11::arg("name"),  pybind11::arg("variable_name") = "",
+             pybind11::arg("separator") = "/", pybind11::return_value_policy::move)
 
         .def("DefineAttribute",
              (adios2::py11::Attribute(adios2::py11::IO::*)(

python/adios2/adios.py

@@ -5,8 +5,12 @@
 
 
 class ADIOS:
-    def __init__(self):
-        self._impl = bindings.ADIOS()
+    def __init__(self, comm=None):
+        if comm:
+            self._impl = bindings.ADIOS(comm)
+        else:
+            self._impl = bindings.ADIOS()
+
         self._operators = {}
         self._ios = {}
 

python/adios2/attribute.py

@@ -13,6 +13,9 @@ def __eq__(self, other):
     def Name(self):
         return self._impl.Name()
 
+    def Type (self):
+        return self._impl.Type()
+
     def Data(self):
         return self._impl.Data()
 

python/adios2/engine.py

@@ -6,8 +6,11 @@
 
 
 class Engine:
-    def __init__(self, io, name, mode):
-        self._impl = io.Open(name, mode)
+    def __init__(self, io, name, mode, comm=None):
+        if comm:
+            self._impl = io.Open(name, mode, comm)
+        else:
+            self._impl = io.Open(name, mode)
 
     def __enter__(self):
         return self

python/adios2/io.py

@@ -24,9 +24,9 @@ def DefineAttribute(
     ):
         return Attribute(self._impl, name, content, variable_name, separator)
 
-    def InquireAttribute(self, name):
+    def InquireAttribute(self, name, variable_name="", separator="/"):
         attr = None
-        attr_impl = self._impl.InquireAttribute(name)
+        attr_impl = self._impl.InquireAttribute(name, variable_name, separator)
         if attr_impl:
             attr = Attribute.__new__(Attribute)
             attr._impl = attr_impl
@@ -51,7 +51,7 @@ def DefineVariable(
         shape=[],
         start=[],
         count=[],
-        isConstantDims=bindings.ConstantDims,
+        isConstantDims=False,
     ):
         if isinstance(content, np.ndarray):
             #if shape is None:
@@ -90,8 +90,8 @@ def RemoveVariable(self, name):
     def RemoveAllVariables(self):
         self._impl.RemoveAllVariables()
 
-    def Open(self, name, mode):
-        return Engine(self._impl, name, mode)
+    def Open(self, name, mode, comm=None):
+        return Engine(self._impl, name, mode, comm)
 
     def SetEngine(self, name):
         self._impl.SetEngine(name)

python/adios2/stream.py

@@ -10,6 +10,7 @@ def open(name, mode, engine_type="BPStream", config_file=None):
 
 def typeAdiosToNumPy(name):
     return {
+        "char": np.int8,
         "int8_t": np.int8,
         "uint8_t": np.uint8,
         "int16_t": np.int16,
@@ -24,9 +25,12 @@ def typeAdiosToNumPy(name):
 
 
 class Stream:
-    def __init__(self, path, mode="r", engine_type="BPStream", config_file=None):
+    def __init__(self, path, mode="r", comm=None, engine_type="BPStream", config_file=None):
         self._io_name = f"stream:{path}:engine_type:{engine_type}"
-        self._adios = ADIOS()
+        if comm:
+            self._adios = ADIOS(comm)
+        else:
+            self._adios = ADIOS()
         self._io = self._adios.DeclareIO(self._io_name)
         self._mode = mode
 
@@ -42,7 +46,7 @@ def __init__(self, path, mode="r", engine_type="BPStream", config_file=None):
         self.max_steps = 0
 
     def __repr__(self):
-        return f"<adios.file named {self._name} and mode {self._mode}>"
+        return f"<adios.file named {self._io_name} and mode {self._mode}>"
 
     def __enter__(self):
         return self
@@ -190,7 +194,7 @@ def write(
         else:
             self._engine.Put(variable, content)
 
-    def read(self, name):
+    def read(self, name, start=None, count=None):
         """
         Random access read allowed to select steps,
         only valid with Stream Engines
@@ -204,16 +208,6 @@ def read(self, name):
 
             count
                 variable local dimensions from offset
-
-            step_start
-                variable step start
-
-            step_count
-                variable number of steps to read from step_start
-
-            block_id
-                required for local array variables
-
         Returns
             array
                 resulting array from selection
@@ -222,6 +216,9 @@ def read(self, name):
         if not variable:
             raise ValueError()
 
+        if start or count:
+            variable.SetSelection([start, count])
+
         if variable.Type() == "string" and variable.SingleValue() is True:
             return self._engine.Get(variable)
 
@@ -231,7 +228,7 @@ def read(self, name):
         self._engine.Get(variable, output)
         return output
 
-    def write_attribute(name, content, variable_name="", separator="/", end_step=False):
+    def write_attribute(self, name, content, variable_name="", separator="/"):
         """
         writes a self-describing single value array (numpy) variable
 
@@ -249,14 +246,41 @@ def write_attribute(name, content, variable_name="", separator="/", end_step=Fal
                 concatenation string between variable_name and attribute
                 e.g. variable_name + separator + name ("var/attr")
                 Not used if variable_name is empty
+        """
+        attribute = self._io.InquireAttribute(name, variable_name, separator)
+        if not attribute:
+            attribute = self._io.DefineAttribute(name, content, variable_name, separator)
 
-            end_step
-                end current step, begin next step and flush
-                (default = false).
+    def read_attribute(self, name, variable_name="", separator="/"):
         """
-        pass
+        Reads a numpy based attribute
+
+        Parameters
+            name
+                attribute name
+
+            variable_name
+                if attribute is associated with a variable
 
-    def read_attribute(name, variable_name, separator="/"):
+            separator
+                concatenation string between variable_name and attribute
+                e.g. variable_name + separator + name (var/attr)
+                Not used if variable_name is empty
+
+        Returns
+            array
+                resulting array attribute data
+        """
+        attribute = self._io.InquireAttribute(name, variable_name, separator)
+        if not attribute:
+            raise KeyError()
+
+        if attribute.Type() == 'string':
+            return attribute.DataString()
+        else:
+            return attribute.Data()
+
+    def read_attribute_string(self, name, variable_name="", separator="/"):
         """
         Reads a numpy based attribute
 
@@ -276,7 +300,11 @@ def read_attribute(name, variable_name, separator="/"):
             array
                 resulting array attribute data
         """
-        pass
+        attribute = self._io.InquireAttribute(name, variable_name, separator)
+        if not attribute:
+            raise KeyError()
+
+        return attribute.DataString()
 
     def begin_step(self):
         self._engine.BeginStep()
@@ -322,3 +350,6 @@ def steps(self, num_steps=0):
 
         self.index = 0
         return self
+
+    def num_steps(self, num_steps=0):
+        return self._engine.Steps()

testing/adios2/python/TestBPWriteTypesHighLevelAPI.py

@@ -12,7 +12,7 @@
 from adios2NPTypes import SmallTestData
 from mpi4py import MPI
 import numpy as np
-from adios2 import *
+from adios2 import Stream, LocalValueDim
 
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
@@ -28,10 +28,10 @@
 
 # Writer
 with Stream("types_np.bp", "w", comm) as s:
-
-    for i in range(0, 5):
-        data.update(rank, i, size)
-        if rank == 0 and i == 0:
+    for step in s.steps(5):
+        data.update(rank, step.current_step(), size)
+        s.write("rank", np.array(rank), shape=[LocalValueDim])
+        if rank == 0 and step.current_step() == 0:
             s.write("tag", "Testing ADIOS2 high-level API")
             s.write("gvarI8", np.array(data.I8[0]))
             s.write("gvarI16", np.array(data.I16[0]))
@@ -70,8 +70,7 @@
             s.write_attribute("attrR32Array", data.R32)
             s.write_attribute("attrR64Array", data.R64)
 
-        s.write("rank", np.array(rank), [adios2.LocalValueDim])
-        s.write("steps", "Step:" + str(i))
+        s.write("steps", "Step:" + str(step.current_step() ))
         s.write("varI8", data.I8, shape, start, count)
         s.write("varI16", data.I16, shape, start, count)
         s.write("varI32", data.I32, shape, start, count)
@@ -83,7 +82,7 @@
         s.write("varR32", data.R32, shape, start, count)
         s.write("varR64", data.R64, shape, start, count)
 
-        if rank == 0 and i == 0:
+        if rank == 0 and step.current_step() == 0:
             s.write_attribute("varattrStrArray", [
                 "varattr1", "varattr2", "varattr3"], "steps")
             s.write_attribute("varattrI8Array", data.I8, "varI8")
@@ -98,8 +97,6 @@
             s.write_attribute("varattrR64Array", data.R64, "varR64")
             s.write_attribute("varattrR64Value", data.R64, "varR64")
 
-        s.end_step()
-
 comm.Barrier()
 
 # Reader
@@ -108,9 +105,9 @@
 with Stream("types_np.bp", "r", comm) as fr:
 
     # file only
-    assert (fr.steps() == 5)
+    assert (fr.num_steps() == 5)
 
-    for fr_step in fr:
+    for fr_step in fr.steps():
 
         step = fr_step.current_step()
         data.update(rank, step, size)
@@ -124,7 +121,7 @@
 #             print("\n")
 
         if step == 0:
-            inTag = fr_step.read_string("tag")
+            inTag = fr_step.read("tag")
             inI8 = fr_step.read("gvarI8")
             inI16 = fr_step.read("gvarI16")
             inI32 = fr_step.read("gvarI32")
@@ -136,7 +133,7 @@
             inR32 = fr_step.read("gvarR32")
             inR64 = fr_step.read("gvarR64")
 
-            if inTag[0] != "Testing ADIOS2 high-level API":
+            if inTag != "Testing ADIOS2 high-level API":
                 print("InTag: " + str(inTag))
                 raise ValueError('tag variable read failed')
 
@@ -171,7 +168,7 @@
                 raise ValueError('gvarR64 read failed')
 
             # attributes
-            inTag = fr_step.read_attribute_string("attrStr")
+            inTag = fr_step.read_attribute("attrStr")
             inI8 = fr_step.read_attribute("attrI8")
             inI16 = fr_step.read_attribute("attrI16")
             inI32 = fr_step.read_attribute("attrI32")
@@ -310,8 +307,8 @@
 
         stepStr = "Step:" + str(step)
 
-        instepStr = fr_step.read_string("steps")
-        if instepStr[0] != stepStr:
+        instepStr = fr_step.read("steps")
+        if instepStr != stepStr:
             raise ValueError('steps variable read failed: ' +
                              instepStr + " " + stepStr)
 
@@ -330,7 +327,6 @@
         indataU64 = fr_step.read("varU64", start, count)
         indataR32 = fr_step.read("varR32", start, count)
         indataR64 = fr_step.read("varR64", start, count)
-        fr_step.end_step()
 
         if (indataI8 == data.I8).all() is False:
             raise ValueError('I8 array read failed')