initial test of NodeSet update (#283)

Can now do the following:

  ns0 = libsonata.NodeSets.from_file('path/to/config_json defined NodeSets')
  ns1 = libsonata.NodeSets.from_file('....')
  duplicates = ns0.update(ns1)

This adds the node sets from ns1 to ns0, and returns the duplicate names.

include/bbp/sonata/node_sets.h

@@ -48,6 +48,15 @@ class SONATA_API NodeSets
  */
  std::set<std::string> names() const;
 
+ /**
+ * Update this NodeSets to include the `other` nodeset
+ *
+ * Duplicate names are overriden with the values from `other`
+ *
+ * The duplicate names are returned.
+ */
+ std::set<std::string> update(const NodeSets& other) const;
+
  /**
  * Return string version of node sets
  */

python/bindings.cpp

@@ -515,6 +515,7 @@ PYBIND11_MODULE(_libsonata, m) {
  .def_static("from_file", [](py::object path) { return NodeSets::fromFile(py::str(path)); })
  .def_property_readonly("names", &NodeSets::names, DOC_NODESETS(names))
  .def("materialize", &NodeSets::materialize, DOC_NODESETS(materialize))
+ .def("update", &NodeSets::update, "other"_a, DOC_NODESETS(update))
  .def("toJSON", &NodeSets::toJSON, DOC_NODESETS(toJSON));
 
  py::class_<CommonPopulationProperties>(m,

python/generated/docstrings.h

@@ -282,6 +282,13 @@ static const char *__doc_bbp_sonata_NodeSets_operator_assign = R"doc()doc";
 
 static const char *__doc_bbp_sonata_NodeSets_toJSON = R"doc(Return string version of node sets)doc";
 
+static const char *__doc_bbp_sonata_NodeSets_update =
+R"doc(Update this NodeSets to include the `other` nodeset
+
+Duplicate names are overriden with the values from `other`
+
+The duplicate names are returned.)doc";
+
 static const char *__doc_bbp_sonata_Population = R"doc()doc";
 
 static const char *__doc_bbp_sonata_PopulationStorage = R"doc(Collection of {PopulationClass}s stored in a H5 file and optional CSV.)doc";
@@ -676,8 +683,6 @@ static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_ampC
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_decayTime = R"doc(The decay time of the bi-exponential shots (ms))doc";
 
-static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_reversal = R"doc(Reversal potential for conductance injection in mV. Default is 0)doc";
-
 static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_dt = R"doc(Timestep of generated signal in ms. Default is 0.25 ms)doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_mean = R"doc(Signal mean in nA (current_clamp) or uS (conductance).)doc";
@@ -686,6 +691,8 @@ static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_rand
 R"doc(Override the random seed to introduce correlations between cells,
 default = None)doc";
 
+static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_reversal = R"doc(Reversal potential for conductance injection in mV. Default is 0)doc";
+
 static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_riseTime = R"doc(The rise time of the bi-exponential shots (ms))doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputAbsoluteShotNoise_sigma = R"doc(signal std dev in nA (current_clamp) or uS (conductance).)doc";
@@ -824,8 +831,6 @@ static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_ampC
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_decayTime = R"doc(The decay time of the bi-exponential shots (ms))doc";
 
-static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_reversal = R"doc(Reversal potential for conductance injection in mV. Default is 0)doc";
-
 static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_dt = R"doc(Timestep of generated signal in ms. Default is 0.25 ms)doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_meanPercent =
@@ -836,6 +841,8 @@ static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_rand
 R"doc(Override the random seed to introduce correlations between cells,
 default = None)doc";
 
+static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_reversal = R"doc(Reversal potential for conductance injection in mV. Default is 0)doc";
+
 static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_riseTime = R"doc(The rise time of the bi-exponential shots (ms))doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputRelativeShotNoise_sdPercent =
@@ -860,8 +867,6 @@ or uS^2 (conductance))doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_decayTime = R"doc(The decay time of the bi-exponential shots (ms))doc";
 
-static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_reversal = R"doc(Reversal potential for conductance injection in mV. Default is 0)doc";
-
 static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_dt = R"doc(Timestep of generated signal in ms. Default is 0.25 ms)doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_randomSeed =
@@ -870,6 +875,8 @@ default = None)doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_rate = R"doc(Rate of Poisson events (Hz))doc";
 
+static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_reversal = R"doc(Reversal potential for conductance injection in mV. Default is 0)doc";
+
 static const char *__doc_bbp_sonata_SimulationConfig_InputShotNoise_riseTime = R"doc(The rise time of the bi-exponential shots (ms))doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_InputSubthreshold = R"doc()doc";
@@ -966,10 +973,10 @@ static const char *__doc_bbp_sonata_SimulationConfig_Report_Type = R"doc()doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_Report_Type_compartment = R"doc()doc";
 
-static const char *__doc_bbp_sonata_SimulationConfig_Report_Type_lfp = R"doc()doc";
-
 static const char *__doc_bbp_sonata_SimulationConfig_Report_Type_invalid = R"doc()doc";
 
+static const char *__doc_bbp_sonata_SimulationConfig_Report_Type_lfp = R"doc()doc";
+
 static const char *__doc_bbp_sonata_SimulationConfig_Report_Type_summation = R"doc()doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_Report_Type_synapse = R"doc()doc";
@@ -1019,6 +1026,8 @@ static const char *__doc_bbp_sonata_SimulationConfig_Run_IntegrationMethod_nicho
 
 static const char *__doc_bbp_sonata_SimulationConfig_Run_dt = R"doc(Integration step duration in milliseconds)doc";
 
+static const char *__doc_bbp_sonata_SimulationConfig_Run_electrodesFile = R"doc(Filename that contains the weights for the LFP calculation.)doc";
+
 static const char *__doc_bbp_sonata_SimulationConfig_Run_integrationMethod =
 R"doc(Selects the NEURON/CoreNEURON integration method. This parameter sets
 the NEURON global variable h.secondorder. Default 0 ('euler'))doc";
@@ -1045,8 +1054,6 @@ is 0.)doc";
 
 static const char *__doc_bbp_sonata_SimulationConfig_Run_tstop = R"doc(Biological simulation end time in milliseconds)doc";
 
-static const char *__doc_bbp_sonata_SimulationConfig_Run_electrodesFile = R"doc(Electrode weights file)doc";
-
 static const char *__doc_bbp_sonata_SimulationConfig_SimulationConfig =
 R"doc(Parses a SONATA JSON simulation configuration file.
 
@@ -1170,19 +1177,27 @@ static const char *__doc_bbp_sonata_SpikeReader_Population_Sorting_by_time = R"d
 
 static const char *__doc_bbp_sonata_SpikeReader_Population_Sorting_none = R"doc()doc";
 
+static const char *__doc_bbp_sonata_SpikeReader_Population_createSpikes = R"doc(Create the spikes from the vectors of node_ids and timestamps)doc";
+
 static const char *__doc_bbp_sonata_SpikeReader_Population_filterNode = R"doc()doc";
 
 static const char *__doc_bbp_sonata_SpikeReader_Population_filterTimestamp = R"doc()doc";
 
 static const char *__doc_bbp_sonata_SpikeReader_Population_get = R"doc(Return spikes with all those node_ids between 'tstart' and 'tstop')doc";
 
+static const char *__doc_bbp_sonata_SpikeReader_Population_getArrays =
+R"doc(Return the node_ids and timestamps vectors with all node_ids between
+'tstart' and 'tstop')doc";
+
+static const char *__doc_bbp_sonata_SpikeReader_Population_getRawArrays = R"doc(Return the raw node_ids and timestamps vectors)doc";
+
 static const char *__doc_bbp_sonata_SpikeReader_Population_getSorting = R"doc(Return the way data are sorted ('none', 'by_id', 'by_time'))doc";
 
 static const char *__doc_bbp_sonata_SpikeReader_Population_getTimes = R"doc(Return (tstart, tstop) of the population)doc";
 
 static const char *__doc_bbp_sonata_SpikeReader_Population_sorting = R"doc()doc";
 
-static const char *__doc_bbp_sonata_SpikeReader_Population_spikes = R"doc()doc";
+static const char *__doc_bbp_sonata_SpikeReader_Population_spike_times = R"doc()doc";
 
 static const char *__doc_bbp_sonata_SpikeReader_Population_tstart = R"doc()doc";
 
@@ -1198,6 +1213,12 @@ static const char *__doc_bbp_sonata_SpikeReader_openPopulation = R"doc()doc";
 
 static const char *__doc_bbp_sonata_SpikeReader_populations = R"doc()doc";
 
+static const char *__doc_bbp_sonata_SpikeTimes = R"doc()doc";
+
+static const char *__doc_bbp_sonata_SpikeTimes_node_ids = R"doc()doc";
+
+static const char *__doc_bbp_sonata_SpikeTimes_timestamps = R"doc()doc";
+
 static const char *__doc_bbp_sonata_detail_NodeSets = R"doc()doc";
 
 static const char *__doc_bbp_sonata_fromValues = R"doc()doc";

python/tests/test_nodesets.py

@@ -181,3 +181,20 @@ def test_library_datatype(self):
 
  j = json.dumps({"NodeSet0": { "E-mapping-good": [0, 1, ] }})
  self.assertRaises(SonataError, NodeSets(j).materialize, "NodeSet0", self.population)
+
+ def test_extend(self):
+ ns0 = NodeSets(json.dumps({"NodeSet0": { "E-mapping-good": 0 }}))
+ dup = ns0.update(ns0)
+ self.assertEqual(dup, {"NodeSet0"})
+ self.assertEqual(ns0.names, {"NodeSet0"})
+
+ ns1 = NodeSets(json.dumps({"NodeSet1": { "E-mapping-good": 0 }}))
+ dup = ns0.update(ns1)
+ self.assertEqual(dup, set())
+ self.assertEqual(ns0.names, {"NodeSet0", "NodeSet1"})
+
+ ns0 = NodeSets(json.dumps({"NodeSet0": { "E-mapping-good": 0 }}))
+ ns1 = NodeSets(json.dumps({"NodeSet0": { "E-mapping-good": 0 }}))
+ dup = ns0.update(ns1)
+ self.assertEqual(dup, {"NodeSet0"})
+ self.assertEqual(ns0.names, {"NodeSet0"})

src/node_sets.cpp

@@ -55,6 +55,7 @@ class NodeSetRule
  virtual bool is_compound() const {
  return false;
  }
+ virtual std::unique_ptr<NodeSetRule> clone() const = 0;
 };
 
 using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
@@ -101,6 +102,20 @@ class NodeSets
  return getMapKeys(node_sets_);
  }
 
+ std::set<std::string> update(const NodeSets& other) {
+ if (&other == this) {
+ return names();
+ }
+ std::set<std::string> duplicates;
+ for (const auto& ns : other.node_sets_) {
+ if (node_sets_.count(ns.first) > 0) {
+ duplicates.insert(ns.first);
+ }
+ node_sets_[ns.first] = ns.second->clone();
+ }
+ return duplicates;
+ }
+
  std::string toJSON() const {
  std::string ret{"{\n"};
  for (const auto& pair : node_sets_) {
@@ -125,15 +140,19 @@ class NodeSetNullRule: public NodeSetRule
  std::string toJSON() const final {
  return {};
  }
+
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetNullRule>();
+ }
 };
 
 // { 'region': ['region1', 'region2', ...] }
 template <typename T>
 class NodeSetBasicRule: public NodeSetRule
 {
  public:
- NodeSetBasicRule(std::string attribute, std::vector<T>& values)
- : attribute_(std::move(attribute))
+ NodeSetBasicRule(const std::string attribute, const std::vector<T>& values)
+ : attribute_(attribute)
  , values_(values) {}
 
  Selection materialize(const detail::NodeSets& /* unused */,
@@ -152,6 +171,10 @@ class NodeSetBasicRule: public NodeSetRule
  return toString(attribute_, values_);
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetBasicRule<T>>(attribute_, values_);
+ }
+
  private:
  std::string attribute_;
  std::vector<T> values_;
@@ -161,7 +184,7 @@ class NodeSetBasicRule: public NodeSetRule
 class NodeSetBasicPopulation: public NodeSetRule
 {
  public:
- explicit NodeSetBasicPopulation(std::vector<std::string>& values)
+ explicit NodeSetBasicPopulation(const std::vector<std::string>& values)
  : values_(values) {}
 
  Selection materialize(const detail::NodeSets& /* unused */,
@@ -177,6 +200,10 @@ class NodeSetBasicPopulation: public NodeSetRule
  return toString("population", values_);
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetBasicPopulation>(values_);
+ }
+
  private:
  std::vector<std::string> values_;
 };
@@ -185,7 +212,7 @@ class NodeSetBasicPopulation: public NodeSetRule
 class NodeSetBasicNodeIds: public NodeSetRule
 {
  public:
- explicit NodeSetBasicNodeIds(Selection::Values&& values)
+ explicit NodeSetBasicNodeIds(const Selection::Values& values)
  : values_(values) {}
 
  Selection materialize(const detail::NodeSets& /* unused */,
@@ -197,6 +224,10 @@ class NodeSetBasicNodeIds: public NodeSetRule
  return toString("node_ids", values_);
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetBasicNodeIds>(values_);
+ }
+
  private:
  Selection::Values values_;
 };
@@ -230,6 +261,15 @@ class NodeSetBasicMultiClause: public NodeSetRule
  return ret;
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ std::vector<NodeSetRulePtr> clauses;
+ clauses.reserve(clauses_.size());
+ for (const auto& clause : clauses_) {
+ clauses.push_back(clause->clone());
+ }
+ return std::make_unique<detail::NodeSetBasicMultiClause>(std::move(clauses));
+ }
+
  private:
  std::vector<NodeSetRulePtr> clauses_;
 };
@@ -238,12 +278,12 @@ class NodeSetBasicMultiClause: public NodeSetRule
 class NodeSetBasicOperatorString: public NodeSetRule
 {
  public:
- explicit NodeSetBasicOperatorString(std::string attribute,
+ explicit NodeSetBasicOperatorString(const std::string& attribute,
  const std::string& op,
- std::string value)
+ const std::string& value)
  : op_(string2op(op))
- , attribute_(std::move(attribute))
- , value_(std::move(value)) {}
+ , attribute_(attribute)
+ , value_(value) {}
 
  Selection materialize(const detail::NodeSets& /* unused */,
  const NodePopulation& np) const final {
@@ -279,6 +319,12 @@ class NodeSetBasicOperatorString: public NodeSetRule
  }
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetBasicOperatorString>(attribute_,
+ op2string(op_),
+ value_);
+ }
+
  private:
  Op op_;
  std::string attribute_;
@@ -349,6 +395,10 @@ class NodeSetBasicOperatorNumeric: public NodeSetRule
  }
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetBasicOperatorNumeric>(name_, op2string(op_), value_);
+ }
+
  private:
  std::string name_;
  double value_;
@@ -382,6 +432,10 @@ class NodeSetCompoundRule: public NodeSetRule
  return targets_;
  }
 
+ std::unique_ptr<NodeSetRule> clone() const final {
+ return std::make_unique<detail::NodeSetCompoundRule>(name_, targets_);
+ }
+
  private:
  std::string name_;
  CompoundTargets targets_;
@@ -488,8 +542,8 @@ NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {
  throw SonataError(
  fmt::format("Operator '{}' must have object with one key value pair", attribute));
  }
- const auto& key = definition.begin().key();
- const auto& value = definition.begin().value();
+ const auto key = definition.begin().key();
+ const auto value = definition.begin().value();
 
  if (value.is_number()) {
  return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,
@@ -673,6 +727,10 @@ std::set<std::string> NodeSets::names() const {
  return impl_->names();
 }
 
+std::set<std::string> NodeSets::update(const NodeSets& other) const {
+ return impl_->update(*other.impl_);
+}
+
 std::string NodeSets::toJSON() const {
  return impl_->toJSON();
 }