From 50746c2724ff1ba1ca0cb1cc7f0200c90dd88ba1 Mon Sep 17 00:00:00 2001 From: David Keeney Date: Fri, 31 May 2019 08:07:40 -0700 Subject: [PATCH 1/9] In progress --- src/CMakeLists.txt | 1 - src/nupic/algorithms/Connections.cpp | 82 ------ src/nupic/algorithms/Connections.hpp | 12 - src/nupic/algorithms/SpatialPooler.cpp | 208 +++---------- src/nupic/algorithms/SpatialPooler.hpp | 70 ++--- src/nupic/algorithms/TemporalMemory.cpp | 274 +++++------------- src/nupic/algorithms/TemporalMemory.hpp | 20 +- .../RandomDistributedScalarEncoder.cpp | 40 +-- .../RandomDistributedScalarEncoder.hpp | 29 +- src/nupic/encoders/ScalarEncoder.cpp | 45 +-- src/nupic/encoders/ScalarEncoder.hpp | 34 ++- src/nupic/engine/Input.cpp | 2 +- src/nupic/engine/Link.cpp | 136 --------- src/nupic/engine/Link.hpp | 3 - src/nupic/engine/Network.cpp | 173 ----------- src/nupic/engine/Network.hpp | 54 ++-- src/nupic/engine/Output.cpp | 2 +- src/nupic/engine/Region.cpp | 138 --------- src/nupic/engine/Region.hpp | 6 +- src/nupic/engine/RegionImpl.cpp | 1 - src/nupic/engine/RegionImpl.hpp | 5 - src/nupic/engine/RegionImplFactory.cpp | 14 - src/nupic/engine/RegionImplFactory.hpp | 4 - src/nupic/engine/RegisteredRegionImpl.hpp | 2 - src/nupic/engine/RegisteredRegionImplCpp.hpp | 4 - src/nupic/engine/Spec.cpp | 68 ++++- src/nupic/engine/Spec.hpp | 4 +- src/nupic/ntypes/ArrayBase.cpp | 42 --- src/nupic/ntypes/ArrayBase.hpp | 4 - src/nupic/ntypes/BundleIO.hpp | 55 ---- src/nupic/ntypes/Dimensions.hpp | 72 +++-- src/nupic/regions/SPRegion.cpp | 83 ------ src/nupic/regions/SPRegion.hpp | 5 - src/nupic/regions/ScalarSensor.cpp | 32 -- src/nupic/regions/ScalarSensor.hpp | 4 - src/nupic/regions/TMRegion.cpp | 101 +------ src/nupic/regions/TMRegion.hpp | 4 - src/nupic/regions/TestNode.cpp | 113 -------- src/nupic/regions/TestNode.hpp | 4 - src/nupic/regions/VectorFile.cpp | 22 -- src/nupic/regions/VectorFile.hpp | 3 - src/nupic/regions/VectorFileEffector.cpp | 9 - src/nupic/regions/VectorFileEffector.hpp | 11 - src/nupic/regions/VectorFileSensor.cpp | 64 ---- src/nupic/regions/VectorFileSensor.hpp | 11 - src/nupic/types/Sdr.cpp | 73 +---- src/nupic/types/Sdr.hpp | 69 +++-- src/nupic/types/Serializable.hpp | 74 ++--- src/nupic/utils/Random.hpp | 5 - .../unit/algorithms/AnomalyLikelihoodTest.cpp | 4 +- src/test/unit/algorithms/ConnectionsTest.cpp | 4 +- .../unit/algorithms/SDRClassifierTest.cpp | 4 +- .../unit/algorithms/SpatialPoolerTest.cpp | 43 ++- .../unit/algorithms/TemporalMemoryTest.cpp | 4 +- src/test/unit/encoders/ScalarEncoderTest.cpp | 9 +- src/test/unit/engine/CppRegionTest.cpp | 4 +- src/test/unit/engine/HelloRegionTest.cpp | 4 +- src/test/unit/engine/LinkTest.cpp | 25 +- src/test/unit/ntypes/ArrayTest.cpp | 2 +- src/test/unit/regions/SPRegionTest.cpp | 8 +- src/test/unit/regions/TMRegionTest.cpp | 8 +- src/test/unit/regions/VectorFileTest.cpp | 4 +- src/test/unit/types/SdrTest.cpp | 6 +- src/test/unit/utils/RandomTest.cpp | 4 +- 64 files changed, 496 insertions(+), 1928 deletions(-) delete mode 100644 src/nupic/ntypes/BundleIO.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9aa30e0873..9f8f90f5b8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -137,7 +137,6 @@ set(ntypes_files nupic/ntypes/ArrayBase.hpp nupic/ntypes/BasicType.cpp nupic/ntypes/BasicType.hpp - nupic/ntypes/BundleIO.hpp nupic/ntypes/Collection.hpp nupic/ntypes/Dimensions.hpp nupic/ntypes/Scalar.cpp diff --git a/src/nupic/algorithms/Connections.cpp b/src/nupic/algorithms/Connections.cpp index 94b4d44bde..b3f7da28b1 100644 --- a/src/nupic/algorithms/Connections.cpp +++ b/src/nupic/algorithms/Connections.cpp @@ -664,88 +664,6 @@ std::ostream& operator<< (std::ostream& stream, const Connections& self) } -void Connections::save(std::ostream &outStream) const { - outStream << std::setprecision(std::numeric_limits::max_digits10); - outStream << std::setprecision(std::numeric_limits::max_digits10); - - // Write a starting marker. - outStream << "Connections" << endl; - outStream << VERSION << endl; - - outStream << cells_.size() << " " << endl; - // Save the original permanence threshold, not the private copy which is used - // only for floating point comparisons. - outStream << connectedThreshold_ + nupic::Epsilon << " " << endl; - - for (CellData cellData : cells_) { - const vector &segments = cellData.segments; - outStream << segments.size() << " "; - - for (Segment segment : segments) { - const SegmentData &segmentData = segments_[segment]; - - const vector &synapses = segmentData.synapses; - outStream << synapses.size() << " "; - - for (Synapse synapse : synapses) { - const SynapseData &synapseData = synapses_[synapse]; - outStream << synapseData.presynapticCell << " "; - outStream << synapseData.permanence << " "; - } - outStream << endl; - } - outStream << endl; - } - outStream << endl; - - outStream << "~Connections" << endl; -} - - -void Connections::load(std::istream &inStream) { - // Check the marker - string marker; - inStream >> marker; - NTA_CHECK(marker == "Connections"); - - // Check the saved version. - int version; - inStream >> version; - NTA_CHECK(version == 2); - - // Retrieve simple variables - UInt numCells; - Permanence connectedThreshold; - inStream >> numCells; - inStream >> connectedThreshold; - initialize(numCells, connectedThreshold); - - for (UInt cell = 0; cell < numCells; cell++) { - - UInt numSegments; - inStream >> numSegments; - - for (SegmentIdx j = 0; j < numSegments; j++) { - Segment segment = createSegment( cell ); - - UInt numSynapses; - inStream >> numSynapses; - - for (SynapseIdx k = 0; k < numSynapses; k++) { - CellIdx presyn; - Permanence perm; - inStream >> presyn; - inStream >> perm; - - createSynapse( segment, presyn, perm ); - } - } - } - - inStream >> marker; - NTA_CHECK(marker == "~Connections"); -} - bool Connections::operator==(const Connections &other) const { if (cells_.size() != other.cells_.size()) diff --git a/src/nupic/algorithms/Connections.hpp b/src/nupic/algorithms/Connections.hpp index f6397c11d7..4cf6928d0a 100644 --- a/src/nupic/algorithms/Connections.hpp +++ b/src/nupic/algorithms/Connections.hpp @@ -468,18 +468,6 @@ class Connections : public Serializable // Serialization - - /** - * Saves serialized data to output stream. - */ - virtual void save(std::ostream &outStream) const override; - - - /** - * Loads serialized data from input stream. - */ - virtual void load(std::istream &inStream) override; - CerealAdapter; template void save_ar(Archive & ar) const { diff --git a/src/nupic/algorithms/SpatialPooler.cpp b/src/nupic/algorithms/SpatialPooler.cpp index 1d2053d281..4df228807c 100644 --- a/src/nupic/algorithms/SpatialPooler.cpp +++ b/src/nupic/algorithms/SpatialPooler.cpp @@ -958,159 +958,6 @@ bool SpatialPooler::isUpdateRound_() const { } -void SpatialPooler::save(ostream &outStream) const { - // Write a starting marker and version. - outStream << std::setprecision(std::numeric_limits::max_digits10); - outStream << "SpatialPooler" << endl; - outStream << version_ << endl; - - // Store the simple variables first. - outStream << numInputs_ << " " << numColumns_ << " " << potentialRadius_ - << " "; - - outStream << potentialPct_ << " "; - outStream << initConnectedPct_ << " " << globalInhibition_ << " " - << numActiveColumnsPerInhArea_ << " " << localAreaDensity_ << " "; - - outStream << stimulusThreshold_ << " " << inhibitionRadius_ << " " - << dutyCyclePeriod_ << " "; - - outStream << boostStrength_ << " "; - - outStream << iterationNum_ << " " << iterationLearnNum_ << " " << spVerbosity_ - << " " << updatePeriod_ << " "; - - outStream << synPermInactiveDec_ << " " - << synPermActiveInc_ << " " << synPermBelowStimulusInc_ << " " - << synPermConnected_ << " " << minPctOverlapDutyCycles_ << " "; - - outStream << wrapAround_ << " " << endl; - - // Store vectors. - outStream << inputDimensions_.size() << " "; - for (auto &elem : inputDimensions_) { - outStream << elem << " "; - } - outStream << endl; - - outStream << columnDimensions_.size() << " "; - for (auto &elem : columnDimensions_) { - outStream << elem << " "; - } - outStream << endl; - - for (UInt i = 0; i < numColumns_; i++) { - outStream << boostFactors_[i] << " "; - } - outStream << endl; - - for (UInt i = 0; i < numColumns_; i++) { - outStream << overlapDutyCycles_[i] << " "; - } - outStream << endl; - - for (UInt i = 0; i < numColumns_; i++) { - outStream << activeDutyCycles_[i] << " "; - } - outStream << endl; - - for (UInt i = 0; i < numColumns_; i++) { - outStream << minOverlapDutyCycles_[i] << " "; - } - outStream << endl; - - for (UInt i = 0; i < numColumns_; i++) { - outStream << tieBreaker_[i] << " "; - } - outStream << endl; - - connections_.save( outStream ); - - //Random - outStream << rng_ << endl; - - outStream << "~SpatialPooler" << endl; -} - -// Implementation note: this method sets up the instance using data from -// inStream. This method does not call initialize. As such we have to be careful -// that everything in initialize is handled properly here. -void SpatialPooler::load(istream &inStream) { - // Current version - version_ = 2; - - // Check the marker - string marker; - inStream >> marker; - NTA_CHECK(marker == "SpatialPooler"); - - // Check the saved version. - UInt version; - inStream >> version; - NTA_CHECK(version == version_); - - // Retrieve simple variables - inStream >> numInputs_ >> numColumns_ >> potentialRadius_ >> potentialPct_ >> - initConnectedPct_ >> globalInhibition_ >> numActiveColumnsPerInhArea_ >> - localAreaDensity_ >> stimulusThreshold_ >> inhibitionRadius_ >> - dutyCyclePeriod_ >> boostStrength_ >> iterationNum_ >> - iterationLearnNum_ >> spVerbosity_ >> updatePeriod_ >> - synPermInactiveDec_ >> synPermActiveInc_ >> synPermBelowStimulusInc_ >> - synPermConnected_ >> minPctOverlapDutyCycles_; - inStream >> wrapAround_; - - // Retrieve vectors. - UInt numInputDimensions; - inStream >> numInputDimensions; - inputDimensions_.resize(numInputDimensions); - for (UInt i = 0; i < numInputDimensions; i++) { - inStream >> inputDimensions_[i]; - } - - UInt numColumnDimensions; - inStream >> numColumnDimensions; - columnDimensions_.resize(numColumnDimensions); - for (UInt i = 0; i < numColumnDimensions; i++) { - inStream >> columnDimensions_[i]; - } - - boostFactors_.resize(numColumns_); - for (UInt i = 0; i < numColumns_; i++) { - inStream >> boostFactors_[i]; - } - - overlapDutyCycles_.resize(numColumns_); - for (UInt i = 0; i < numColumns_; i++) { - inStream >> overlapDutyCycles_[i]; - } - - activeDutyCycles_.resize(numColumns_); - for (UInt i = 0; i < numColumns_; i++) { - inStream >> activeDutyCycles_[i]; - } - - minOverlapDutyCycles_.resize(numColumns_); - for (UInt i = 0; i < numColumns_; i++) { - inStream >> minOverlapDutyCycles_[i]; - } - - tieBreaker_.resize(numColumns_); - for (UInt i = 0; i < numColumns_; i++) { - inStream >> tieBreaker_[i]; - } - - connections_.load( inStream ); - - inStream >> rng_; - - inStream >> marker; - NTA_CHECK(marker == "~SpatialPooler"); - - // initialize ephemeral members - overlaps_.resize(numColumns_); - overlapsPct_.resize(numColumns_); - boostedOverlaps_.resize(numColumns_); -} //---------------------------------------------------------------------- @@ -1118,10 +965,9 @@ void SpatialPooler::load(istream &inStream) { //---------------------------------------------------------------------- // Print the main SP creation parameters -void SpatialPooler::printParameters() const { - std::cout << "------------CPP SpatialPooler Parameters ------------------\n"; - std::cout - << "iterationNum = " << getIterationNum() << std::endl +void SpatialPooler::printParameters(std::ostream& out) const { + out << "------------CPP SpatialPooler Parameters ------------------\n"; + out << "iterationNum = " << getIterationNum() << std::endl << "iterationLearnNum = " << getIterationLearnNum() << std::endl << "numInputs = " << getNumInputs() << std::endl << "numColumns = " << getNumColumns() << std::endl @@ -1144,40 +990,62 @@ void SpatialPooler::printParameters() const { << "version = " << version() << std::endl; } -void SpatialPooler::printState(vector &state) { - std::cout << "[ "; +void SpatialPooler::printState(const vector &state, std::ostream& out) const { + out << "[ "; for (UInt i = 0; i != state.size(); ++i) { if (i > 0 && i % 10 == 0) { - std::cout << "\n "; + out << "\n "; } - std::cout << state[i] << " "; + out << state[i] << " "; } - std::cout << "]\n"; + out << "]\n"; } -void SpatialPooler::printState(vector &state) { - std::cout << "[ "; +void SpatialPooler::printState(const vector &state, std::ostream& out) const { + out << "[ "; for (UInt i = 0; i != state.size(); ++i) { if (i > 0 && i % 10 == 0) { - std::cout << "\n "; + out << "\n "; } - std::printf("%6.3f ", state[i]); + out << state[i]; } - std::cout << "]\n"; + out << "]\n"; } -/** equals implementation based on serialization */ +std::ostream & nupic::algorithms::spatial_pooler::operator<<(std::ostream & out, const SpatialPooler &sp) +{ + sp.printParameters(out); + out << "inputDimensions: "; + sp.printState(sp.getInputDimensions(), out); + out << "columnDimensions: "; + sp.printState(sp.getColumnDimensions(), out); + out << "boostFactors: "; + sp.printState(sp.boostFactors_, out); + out << "overlapDutyCycles: "; + sp.printState(sp.overlapDutyCycles_, out); + out << "activeDutyCycles: "; + sp.printState(sp.activeDutyCycles_, out); + out << "minOverlapDutyCycles: "; + sp.printState(sp.minOverlapDutyCycles_, out); + out << "tieBreaker: "; + sp.printState(sp.tieBreaker_, out); + out << "Connections: " << sp.connections_; + return out; +} + +/** equals implementation based on text serialization */ bool SpatialPooler::operator==(const SpatialPooler& o) const{ stringstream s; s.flags(ios::scientific); s.precision(numeric_limits::digits10 + 1); - this->save(s); + s << *this; const string thisStr = s.str(); s.str(""); //clear stream - o.save(s); + s << o; const string otherStr = s.str(); return thisStr == otherStr; } + diff --git a/src/nupic/algorithms/SpatialPooler.hpp b/src/nupic/algorithms/SpatialPooler.hpp index 8f7aa82d83..17150c0e76 100644 --- a/src/nupic/algorithms/SpatialPooler.hpp +++ b/src/nupic/algorithms/SpatialPooler.hpp @@ -258,26 +258,17 @@ class SpatialPooler : public Serializable virtual UInt version() const { return version_; }; /** - Save (serialize) the current state of the spatial pooler to the - specified file. + save_ar()/load_ar() Serialize the current state of the spatial pooler to the + specified file and deserialize it. - @param fd A valid file descriptor. + @param Archive& ar See Serializable.hpp */ - virtual void save(ostream &outStream) const override; - - - /** - Load (deserialize) and initialize the spatial pooler from the - specified input stream. - - @param inStream A valid istream. - */ - virtual void load(istream &inStream) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template void save_ar(Archive& ar) const { + ar(CEREAL_NVP(inputDimensions_), + CEREAL_NVP(columnDimensions_)); ar(CEREAL_NVP(numInputs_), CEREAL_NVP(numColumns_), CEREAL_NVP(potentialRadius_), @@ -299,20 +290,20 @@ class SpatialPooler : public Serializable CEREAL_NVP(synPermBelowStimulusInc_), CEREAL_NVP(synPermConnected_), CEREAL_NVP(minPctOverlapDutyCycles_), - CEREAL_NVP(wrapAround_), - CEREAL_NVP(inputDimensions_), - CEREAL_NVP(columnDimensions_), - CEREAL_NVP(boostFactors_), - CEREAL_NVP(overlapDutyCycles_), - CEREAL_NVP(activeDutyCycles_), - CEREAL_NVP(minOverlapDutyCycles_), - CEREAL_NVP(tieBreaker_)); + CEREAL_NVP(wrapAround_)); + ar(CEREAL_NVP(boostFactors_)); + ar(CEREAL_NVP(overlapDutyCycles_)); + ar(CEREAL_NVP(activeDutyCycles_)); + ar(CEREAL_NVP(minOverlapDutyCycles_)); + ar(CEREAL_NVP(tieBreaker_)); ar(CEREAL_NVP(connections_)); ar(CEREAL_NVP(rng_)); } // FOR Cereal Deserialization template void load_ar(Archive& ar) { + ar(CEREAL_NVP(inputDimensions_), + CEREAL_NVP(columnDimensions_)); ar(CEREAL_NVP(numInputs_), CEREAL_NVP(numColumns_), CEREAL_NVP(potentialRadius_), @@ -335,13 +326,11 @@ class SpatialPooler : public Serializable CEREAL_NVP(synPermConnected_), CEREAL_NVP(minPctOverlapDutyCycles_), CEREAL_NVP(wrapAround_)); - ar(CEREAL_NVP(inputDimensions_), - CEREAL_NVP(columnDimensions_)); - ar(CEREAL_NVP(boostFactors_), - CEREAL_NVP(overlapDutyCycles_), - CEREAL_NVP(activeDutyCycles_), - CEREAL_NVP(minOverlapDutyCycles_), - CEREAL_NVP(tieBreaker_)); + ar(CEREAL_NVP(boostFactors_)); + ar(CEREAL_NVP(overlapDutyCycles_)); + ar(CEREAL_NVP(activeDutyCycles_)); + ar(CEREAL_NVP(minOverlapDutyCycles_)); + ar(CEREAL_NVP(tieBreaker_)); ar(CEREAL_NVP(connections_)); ar(CEREAL_NVP(rng_)); @@ -782,15 +771,6 @@ class SpatialPooler : public Serializable */ void getConnectedCounts(UInt connectedCounts[]) const; - /** - Print the main SP creation parameters to stdout. - */ - void printParameters() const; - - friend std::ostream& operator<< (std::ostream& stream, const SpatialPooler& self) { - stream << "SpatialPooler " << self.connections_; - return stream; - } /** Returns the overlap score for each column. @@ -1202,11 +1182,19 @@ class SpatialPooler : public Serializable /** Print the given UInt array in a nice format */ - void printState(vector &state); + void printState(const vector &state, std::ostream& out=std::cout) const ; /** Print the given Real array in a nice format */ - void printState(vector &state); + void printState(const vector &state, std::ostream& out=std::cout) const; + + /** + Print the main SP creation parameters to stdout. + */ + void printParameters(std::ostream& out=std::cout) const; + + friend std::ostream& operator<< (std::ostream& stream, const SpatialPooler& self); + protected: UInt numInputs_; @@ -1263,6 +1251,8 @@ class SpatialPooler : public Serializable const connections::Connections &connections = connections_; }; + + } // end namespace spatial_pooler } // end namespace algorithms } // end namespace nupic diff --git a/src/nupic/algorithms/TemporalMemory.cpp b/src/nupic/algorithms/TemporalMemory.cpp index f2edb2db44..bb9db54937 100644 --- a/src/nupic/algorithms/TemporalMemory.cpp +++ b/src/nupic/algorithms/TemporalMemory.cpp @@ -771,210 +771,6 @@ SynapseIdx TemporalMemory::getMaxSynapsesPerSegment() const { UInt TemporalMemory::version() const { return TM_VERSION; } -template -static void saveFloat_(ostream &outStream, FloatType v) { - outStream << std::setprecision(std::numeric_limits::max_digits10) - << v << " "; -} - -void TemporalMemory::save(ostream &outStream) const { - // Write a starting marker and version. - outStream << "TemporalMemory" << endl; - outStream << TM_VERSION << endl; - - outStream << numColumns_ << " " << cellsPerColumn_ << " " - << activationThreshold_ << " "; - - saveFloat_(outStream, initialPermanence_); - saveFloat_(outStream, connectedPermanence_); - - outStream << minThreshold_ << " " << maxNewSynapseCount_ << " " - << checkInputs_ << " "; - - saveFloat_(outStream, permanenceIncrement_); - saveFloat_(outStream, permanenceDecrement_); - saveFloat_(outStream, predictedSegmentDecrement_); - saveFloat_(outStream, anomaly_); - - outStream << extra_ << " "; - outStream << maxSegmentsPerCell_ << " " << maxSynapsesPerSegment_ << " " - << iteration_ << " "; - - outStream << endl; - - connections.save(outStream); - outStream << endl; - - outStream << rng_ << endl; - - outStream << columnDimensions_.size() << " "; - for (auto &elem : columnDimensions_) { - outStream << elem << " "; - } - outStream << endl; - - outStream << activeCells_.size() << " "; - for (CellIdx cell : activeCells_) { - outStream << cell << " "; - } - outStream << endl; - - outStream << winnerCells_.size() << " "; - for (CellIdx cell : winnerCells_) { - outStream << cell << " "; - } - outStream << endl; - - outStream << segmentsValid_ << " "; - outStream << activeSegments_.size() << " "; - for (Segment segment : activeSegments_) { - const CellIdx cell = connections.cellForSegment(segment); - const vector &segments = connections.segmentsForCell(cell); - - SegmentIdx idx = (SegmentIdx)std::distance( - segments.begin(), std::find(segments.begin(), segments.end(), segment)); - - outStream << idx << " "; - outStream << cell << " "; - outStream << numActiveConnectedSynapsesForSegment_[segment] << " "; - } - outStream << endl; - - outStream << matchingSegments_.size() << " "; - for (Segment segment : matchingSegments_) { - const CellIdx cell = connections.cellForSegment(segment); - const vector &segments = connections.segmentsForCell(cell); - - SegmentIdx idx = (SegmentIdx)std::distance( - segments.begin(), std::find(segments.begin(), segments.end(), segment)); - - outStream << idx << " "; - outStream << cell << " "; - outStream << numActivePotentialSynapsesForSegment_[segment] << " "; - } - outStream << endl; - - outStream << "~TemporalMemory" << endl; -} - - - -void TemporalMemory::load(istream &inStream) { - // Check the marker - string marker; - inStream >> marker; - NTA_CHECK(marker == "TemporalMemory"); - - // Check the saved version. - UInt version; - inStream >> version; - NTA_CHECK(version <= TM_VERSION); - - // Retrieve simple variables - inStream >> numColumns_ >> cellsPerColumn_ >> activationThreshold_ >> - initialPermanence_ >> connectedPermanence_ >> minThreshold_ >> - maxNewSynapseCount_ >> checkInputs_ >> permanenceIncrement_ >> - permanenceDecrement_ >> predictedSegmentDecrement_ >> anomaly_ >> extra_ >> - maxSegmentsPerCell_ >> maxSynapsesPerSegment_ >> iteration_; - - connections.load(inStream); - - numActiveConnectedSynapsesForSegment_.assign( - connections.segmentFlatListLength(), 0); - numActivePotentialSynapsesForSegment_.assign( - connections.segmentFlatListLength(), 0); - - inStream >> rng_; - - // Retrieve vectors. - UInt numColumnDimensions; - inStream >> numColumnDimensions; - columnDimensions_.resize(numColumnDimensions); - for (UInt i = 0; i < numColumnDimensions; i++) { - inStream >> columnDimensions_[i]; - } - - UInt numActiveCells; - inStream >> numActiveCells; - for (UInt i = 0; i < numActiveCells; i++) { - CellIdx cell; - inStream >> cell; - activeCells_.push_back(cell); - } - - if (version < 2) { - UInt numPredictiveCells; - inStream >> numPredictiveCells; - for (UInt i = 0; i < numPredictiveCells; i++) { - CellIdx cell; - inStream >> cell; // Ignore - } - } - - UInt numWinnerCells; - inStream >> numWinnerCells; - for (UInt i = 0; i < numWinnerCells; i++) { - CellIdx cell; - inStream >> cell; - winnerCells_.push_back(cell); - } - - inStream >> segmentsValid_; - UInt numActiveSegments; - inStream >> numActiveSegments; - activeSegments_.resize(numActiveSegments); - for (UInt i = 0; i < numActiveSegments; i++) { - SegmentIdx idx; - inStream >> idx; - - CellIdx cellIdx; - inStream >> cellIdx; - - Segment segment = connections.getSegment(cellIdx, idx); - activeSegments_[i] = segment; - - if (version < 2) { - numActiveConnectedSynapsesForSegment_[segment] = 0; // Unknown - } else { - inStream >> numActiveConnectedSynapsesForSegment_[segment]; - } - } - - UInt numMatchingSegments; - inStream >> numMatchingSegments; - matchingSegments_.resize(numMatchingSegments); - for (UInt i = 0; i < numMatchingSegments; i++) { - SegmentIdx idx; - inStream >> idx; - - CellIdx cellIdx; - inStream >> cellIdx; - - Segment segment = connections.getSegment(cellIdx, idx); - matchingSegments_[i] = segment; - - if (version < 2) { - numActivePotentialSynapsesForSegment_[segment] = 0; // Unknown - } else { - inStream >> numActivePotentialSynapsesForSegment_[segment]; - } - } - - if (version < 2) { - UInt numMatchingCells; - inStream >> numMatchingCells; - for (UInt i = 0; i < numMatchingCells; i++) { - CellIdx cell; - inStream >> cell; // Ignore - } - } - - lastUsedIterationForSegment_.resize(connections.segmentFlatListLength()); - - inStream >> marker; - NTA_CHECK(marker == "~TemporalMemory"); -} - static set> getComparableSegmentSet(const Connections &connections, const vector &segments) { @@ -1026,10 +822,9 @@ bool TemporalMemory::operator==(const TemporalMemory &other) const { //---------------------------------------------------------------------- // Print the main TM creation parameters -void TemporalMemory::printParameters() { - std::cout << "------------CPP TemporalMemory Parameters ------------------\n"; - std::cout - << "version = " << TM_VERSION << std::endl +void TemporalMemory::printParameters(std::ostream& out) const { + out << "------------CPP TemporalMemory Parameters ------------------\n"; + out << "version = " << TM_VERSION << std::endl << "numColumns = " << numberOfColumns() << std::endl << "cellsPerColumn = " << getCellsPerColumn() << std::endl << "activationThreshold = " << getActivationThreshold() << std::endl @@ -1037,11 +832,74 @@ void TemporalMemory::printParameters() { << "connectedPermanence = " << getConnectedPermanence() << std::endl << "minThreshold = " << getMinThreshold() << std::endl << "maxNewSynapseCount = " << getMaxNewSynapseCount() << std::endl + << "checkInputs = " << checkInputs_ << std::endl << "permanenceIncrement = " << getPermanenceIncrement() << std::endl << "permanenceDecrement = " << getPermanenceDecrement() << std::endl << "predictedSegmentDecrement = " << getPredictedSegmentDecrement() + << "anomaly = " << anomaly_ << std::endl << std::endl << "maxSegmentsPerCell = " << getMaxSegmentsPerCell() << std::endl << "maxSynapsesPerSegment = " << getMaxSynapsesPerSegment() + << "extra = " << extra_ << std::endl + << "iteration = " << iteration_ << std::endl << std::endl; } + + +std::ostream& nupic::algorithms::temporal_memory::operator<< (std::ostream& out, const TemporalMemory& tm) { + tm.printParameters(out); + + out << "Connections: " << tm.connections << endl; + out << "rng: " << tm.rng_ << endl; + + out << "columnDimensions: [ "; + for (auto &elem : tm.columnDimensions_) { + out << elem << " "; + } + out << "]" << endl; + + out << "activeCells: [ "; + for (CellIdx cell : tm.activeCells_) { + out << cell << " "; + } + out << "]" << endl; + + out << "winnerCells: [ "; + for (CellIdx cell : tm.winnerCells_) { + out << cell << " "; + } + out << "]" << endl << endl; + + out << "segmentsValid: " << tm.segmentsValid_ << " "; + out << "activeSegments: ["; + for (Segment segment : tm.activeSegments_) { + const CellIdx cell = tm.connections.cellForSegment(segment); + const vector &segments = tm.connections.segmentsForCell(cell); + + SegmentIdx idx = (SegmentIdx)std::distance( + segments.begin(), std::find(segments.begin(), segments.end(), segment)); + + out << " [" << endl; + out << idx << ","; + out << cell << ","; + out << tm.numActiveConnectedSynapsesForSegment_[segment] << "]" << endl; + } + out << "]" << endl << endl; + + out << "matchingSegments: [" << endl; + for (Segment segment : tm.matchingSegments_) { + const CellIdx cell = tm.connections.cellForSegment(segment); + const vector &segments = tm.connections.segmentsForCell(cell); + + SegmentIdx idx = (SegmentIdx)std::distance( + segments.begin(), std::find(segments.begin(), segments.end(), segment)); + + out << " [" << endl; + out << idx << ","; + out << cell << ","; + out << tm.numActivePotentialSynapsesForSegment_[segment] << "]" << endl; + } + out << "]" << endl << endl; + return out; +} + diff --git a/src/nupic/algorithms/TemporalMemory.hpp b/src/nupic/algorithms/TemporalMemory.hpp index a8004178a8..6518bb5c7a 100644 --- a/src/nupic/algorithms/TemporalMemory.hpp +++ b/src/nupic/algorithms/TemporalMemory.hpp @@ -435,22 +435,11 @@ using namespace nupic::algorithms::connections; SynapseIdx getMaxSynapsesPerSegment() const; /** - * Save (serialize) the current state of the spatial pooler to the - * specified file. + * Save (serialize) / Load (deserialize) the current state of the spatial pooler + * to the specified stream. * - * @param fd A valid file descriptor. + * @param Archive & ar a Cereal container. */ - virtual void save(ostream &outStream) const override; - - - /** - * Load (deserialize) and initialize the spatial pooler from the - * specified input stream. - * - * @param inStream A valid istream. - */ - virtual void load(istream &inStream) override; - // a container to hold the data for one sequence item during serialization struct container_ar { SegmentIdx idx; @@ -584,6 +573,7 @@ using namespace nupic::algorithms::connections; virtual bool operator==(const TemporalMemory &other) const; inline bool operator!=(const TemporalMemory &other) const { return not this->operator==(other); } + friend std::ostream& operator<< (std::ostream& stream, const TemporalMemory& self); //---------------------------------------------------------------------- // Debugging helpers @@ -592,7 +582,7 @@ using namespace nupic::algorithms::connections; /** * Print the main TM creation parameters */ - void printParameters(); + void printParameters(std::ostream& out=std::cout) const; /** * Returns the index of the (mini-)column that a cell belongs to. diff --git a/src/nupic/encoders/RandomDistributedScalarEncoder.cpp b/src/nupic/encoders/RandomDistributedScalarEncoder.cpp index e5d5578fcb..5b7e2ba658 100644 --- a/src/nupic/encoders/RandomDistributedScalarEncoder.cpp +++ b/src/nupic/encoders/RandomDistributedScalarEncoder.cpp @@ -127,37 +127,13 @@ void RandomDistributedScalarEncoder::encode(Real64 input, sdr::SDR &output) output.setDense( data ); } -void RandomDistributedScalarEncoder::save(std::ostream &stream) const +std::ostream & nupic::encoders::operator<<(std::ostream & out, const RandomDistributedScalarEncoder &self) { - stream << "RDSE "; - stream << parameters.size << " "; - stream << parameters.activeBits << " "; - stream << parameters.resolution << " "; - stream << parameters.category << " "; - stream << parameters.seed << " "; - stream << "~RDSE~" << endl; -} - -void RandomDistributedScalarEncoder::load(std::istream &stream) -{ - string prelude; - stream >> prelude; - NTA_CHECK( prelude == "RDSE" ); - - RDSE_Parameters p; - stream >> p.size; - stream >> p.activeBits; - stream >> p.resolution; - stream >> p.category; - stream >> p.seed; - - string postlude; - stream >> postlude; - NTA_CHECK( postlude == "~RDSE~" ); - stream.ignore( 1 ); // Eat the trailing newline. - - if( p.category ) { - p.resolution = 0.0f; - } - initialize( p ); + out << "RDSE "; + out << "size: " << self.parameters.size << ", "; + out << "activeBits: " << self.parameters.activeBits << ", "; + out << "resolution: " << self.parameters.resolution << ", "; + out << "category: " << self.parameters.category << ", "; + out << "seed: " << self.parameters.seed << std::endl; + return out; } diff --git a/src/nupic/encoders/RandomDistributedScalarEncoder.hpp b/src/nupic/encoders/RandomDistributedScalarEncoder.hpp index 4105c64491..d96e2727f0 100644 --- a/src/nupic/encoders/RandomDistributedScalarEncoder.hpp +++ b/src/nupic/encoders/RandomDistributedScalarEncoder.hpp @@ -25,6 +25,7 @@ #define NTA_ENCODERS_RDSE #include +#include namespace nupic { namespace encoders { @@ -122,17 +123,41 @@ class RandomDistributedScalarEncoder : public BaseEncoder void encode(Real64 input, sdr::SDR &output) override; - void save(std::ostream &stream) const override; - void load(std::istream &stream) override; ~RandomDistributedScalarEncoder() override {}; + CerealAdapter; // see Serializable.hpp + // FOR Cereal Serialization + template + void save_ar(Archive& ar) const { + std::string name = "RandomDistributedScalarEncoder"; + ar(cereal::make_nvp("name", name)); + ar(cereal::make_nvp("size", args_.size)); + ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("resolution", args_.resolution)); + ar(cereal::make_nvp("category", args_.category)); + ar(cereal::make_nvp("seed", args_.seed)); + } + + // FOR Cereal Deserialization + template + void load_ar(Archive& ar) { + std::string name; + ar(cereal::make_nvp("name", name)); + ar(cereal::make_nvp("size", args_.size)); + ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("resolution", args_.resolution)); + ar(cereal::make_nvp("category", args_.category)); + ar(cereal::make_nvp("seed", args_.seed)); + } private: RDSE_Parameters args_; }; typedef RandomDistributedScalarEncoder RDSE; +std::ostream & operator<<(std::ostream & out, const RandomDistributedScalarEncoder & self); + } // End namespace encoders } // End namespace nupic #endif // End ifdef NTA_ENCODERS_RDSE diff --git a/src/nupic/encoders/ScalarEncoder.cpp b/src/nupic/encoders/ScalarEncoder.cpp index 0191701320..ec9b8406b3 100644 --- a/src/nupic/encoders/ScalarEncoder.cpp +++ b/src/nupic/encoders/ScalarEncoder.cpp @@ -195,45 +195,20 @@ void ScalarEncoder::encode(Real64 input, SDR &output) output.setSparse( sparse ); } -void ScalarEncoder::save(std::ostream &stream) const +std::ostream & nupic::encoders::operator<<(std::ostream & out, const ScalarEncoder &self) { - stream << "ScalarEncoder "; - stream << parameters.minimum << " "; - stream << parameters.maximum << " "; - stream << parameters.clipInput << " "; - stream << parameters.periodic << " "; - stream << parameters.category << " "; - stream << parameters.activeBits << " "; + out << "ScalarEncoder "; + out << "minimum: " << self.parameters.minimum << ", "; + out << "maximum: " << self.parameters.maximum << ", "; + out << "clipInput: " << self.parameters.clipInput << ", "; + out << "periodic: " << self.parameters.periodic << ", "; + out << "category: " << self.parameters.category << ", "; + out << "activeBits: " << self.parameters.activeBits << ", "; // Save the resolution instead of the size BC it's higher precision. - stream << parameters.resolution << " "; - stream << "~ScalarEncoder~" << std::endl; + out << "resolution: " << self.parameters.resolution << std::endl; + return out; } -void ScalarEncoder::load(std::istream &stream) -{ - std::string prelude; - stream >> prelude; - NTA_CHECK( prelude == "ScalarEncoder" ); - - ScalarEncoderParameters p; - stream >> p.minimum; - stream >> p.maximum; - stream >> p.clipInput; - stream >> p.periodic; - stream >> p.category; - stream >> p.activeBits; - stream >> p.resolution; - - std::string postlude; - stream >> postlude; - NTA_CHECK( postlude == "~ScalarEncoder~" ); - stream.ignore( 1 ); // Eat the trailing newline. - - if( p.category ) { - p.resolution = 0.0f; - } - initialize( p ); -} } // end namespace encoders } // end namespace nupic diff --git a/src/nupic/encoders/ScalarEncoder.hpp b/src/nupic/encoders/ScalarEncoder.hpp index 8bd700ceef..957c2536c1 100644 --- a/src/nupic/encoders/ScalarEncoder.hpp +++ b/src/nupic/encoders/ScalarEncoder.hpp @@ -137,14 +137,44 @@ namespace encoders { void encode(Real64 input, sdr::SDR &output) override; - void save(std::ostream &stream) const override; - void load(std::istream &stream) override; + + CerealAdapter; // see Serializable.hpp + // FOR Cereal Serialization + template + void save_ar(Archive& ar) const { + std::string name = "ScalarEncoder"; + ar(cereal::make_nvp("name", name)); + ar(cereal::make_nvp("minimum", args_.minimum)); + ar(cereal::make_nvp("maximum", args_.maximum)); + ar(cereal::make_nvp("clipInput", args_.clipInput)); + ar(cereal::make_nvp("periodic", args_.periodic)); + ar(cereal::make_nvp("category", args_.category)); + ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("resolution", args_.resolution)); + } + + // FOR Cereal Deserialization + template + void load_ar(Archive& ar) { + std::string name; + ar(cereal::make_nvp("name", name)); + ar(cereal::make_nvp("minimum", args_.minimum)); + ar(cereal::make_nvp("maximum", args_.maximum)); + ar(cereal::make_nvp("clipInput", args_.clipInput)); + ar(cereal::make_nvp("periodic", args_.periodic)); + ar(cereal::make_nvp("category", args_.category)); + ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("resolution", args_.resolution)); + } ~ScalarEncoder() override {}; private: ScalarEncoderParameters args_; }; // end class ScalarEncoder + + std::ostream & operator<<(std::ostream & out, const ScalarEncoder &self); + } // end namespace encoders } // end namespace nupic #endif // end NTA_ENCODERS_SCALAR diff --git a/src/nupic/engine/Input.cpp b/src/nupic/engine/Input.cpp index e4df01067a..7bb363953d 100644 --- a/src/nupic/engine/Input.cpp +++ b/src/nupic/engine/Input.cpp @@ -311,7 +311,7 @@ void Input::initialize() { dim_ = inD; if (data_.getType() == NTA_BasicType_SDR) { - data_.allocateBuffer(dim_); + data_.allocateBuffer(dim_.asVector()); } else if (dim_.isDontcare()) { data_.allocateBuffer(0); // lets hope this is an unused input. } else { diff --git a/src/nupic/engine/Link.cpp b/src/nupic/engine/Link.cpp index ab347e0743..3ae4e503a0 100644 --- a/src/nupic/engine/Link.cpp +++ b/src/nupic/engine/Link.cpp @@ -232,142 +232,6 @@ void Link::shiftBufferedData() { } } -void Link::serialize(std::ostream &f) { - size_t srcCount = ((!src_) ? (size_t)0 : src_->getData().getCount()); - - f << "{\n"; - f << "linkType: " << getLinkType() << "\n"; - f << "params: " << getLinkParams() << "\n"; - f << "srcRegion: " << getSrcRegionName() << "\n"; - f << "srcOutput: " << getSrcOutputName() << "\n"; - f << "destRegion: " << getDestRegionName() << "\n"; - f << "destInput: " << getDestInputName() << "\n"; - f << "propagationDelay: " << propagationDelay_ << "\n"; - f << "propagationDelayBuffer: [ " << propagationDelayBuffer_.size() << "\n"; - if (propagationDelay_ > 0) { - // we need to capture the propagationDelayBuffer_ used for propagationDelay - // Do not copy the last entry. It is the same as the output buffer. - - // The current contents of the Destination Input buffer also needs - // to be captured as if it were the top value of the propagationDelayBuffer. - // When restored, it will be copied to the dest input buffer and popped off - // before the next execution. If there is an offset, we only - // want to capture the amount of the input buffer contributed by - // this link. - Array a = dest_->getData().subset(destOffset_, srcCount); - a.save(f); // our part of the current Dest Input buffer. - - std::deque::iterator itr; - for (auto itr = propagationDelayBuffer_.begin(); - itr != propagationDelayBuffer_.end(); itr++) { - if (itr + 1 == propagationDelayBuffer_.end()) - break; // skip the last buffer. Its the current output. - Array &buf = *itr; - buf.save(f); - } // end for - } - f << "]\n"; // end of list of buffers in propagationDelayBuffer - - f << "}\n"; // end of sequence -} - -void Link::deserialize(std::istream &f) { - // Each link is a map -- extract the 9 values in the map - // The "circularBuffer" element is a two dimentional array only present if - // propogationDelay > 0. - char bigbuffer[5000]; - std::string tag; - Size count; - std::string linkType; - std::string linkParams; - std::string srcRegionName; - std::string srcOutputName; - std::string destRegionName; - std::string destInputName; - Size propagationDelay; - - f >> tag; - NTA_CHECK(tag == "{") << "Invalid network structure file -- bad link (not a map)"; - - // 1. type - f >> tag; - NTA_CHECK(tag == "linkType:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - linkType = bigbuffer; - - // 2. params - f >> tag; - NTA_CHECK(tag == "params:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - linkParams = bigbuffer; - - // 3. srcRegion (name) - f >> tag; - NTA_CHECK(tag == "srcRegion:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - srcRegionName = bigbuffer; - - // 4. srcOutput - f >> tag; - NTA_CHECK(tag == "srcOutput:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - srcOutputName = bigbuffer; - - // 5. destRegion - f >> tag; - NTA_CHECK(tag == "destRegion:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - destRegionName = bigbuffer; - - // 6. destInput - f >> tag; - NTA_CHECK(tag == "destInput:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - destInputName = bigbuffer; - - - - // 7. propagationDelay (number of cycles to delay propagation) - f >> tag; - NTA_CHECK(tag == "propagationDelay:"); - f >> propagationDelay; - - // fill in the data for the Link object - commonConstructorInit_(linkType, linkParams, srcRegionName, destRegionName, - srcOutputName, destInputName, propagationDelay); - - // 8. propagationDelayBuffer - f >> tag; - NTA_CHECK(tag == "propagationDelayBuffer:"); - f >> tag; - NTA_CHECK(tag == "[") << "Expected start of a sequence."; - f >> count; - f.ignore(1); - // if no propagationDelay (value = 0) then there should be an empty sequence. - NTA_CHECK(count == propagationDelay_) << "Invalid network structure file -- " - "link has " << count << " buffers in 'propagationDelayBuffer'. " - << "Expecting " << propagationDelay << "."; - Size idx = 0; - for (; idx < count; idx++) { - Array a; - a.load(f); - propagationDelayBuffer_.push_back(a); - } - // To complete the restore, call r->prepareInputs() and then shiftBufferedData(); - // This is performed in Network class at the end of the load(). - f >> tag; - NTA_CHECK(tag == "]"); - f >> tag; - NTA_CHECK(tag == "}"); - f.ignore(1); -} - bool Link::operator==(const Link &o) const { diff --git a/src/nupic/engine/Link.hpp b/src/nupic/engine/Link.hpp index d3200ad718..bc75327ea3 100644 --- a/src/nupic/engine/Link.hpp +++ b/src/nupic/engine/Link.hpp @@ -469,9 +469,6 @@ class Link : public Serializable /** * Serialize/Deserialize the link. */ - void serialize(std::ostream &f); // TODO:cereal Remove - void deserialize(std::istream &f); - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template diff --git a/src/nupic/engine/Network.cpp b/src/nupic/engine/Network.cpp index 384e0ccd91..28597d4a03 100644 --- a/src/nupic/engine/Network.cpp +++ b/src/nupic/engine/Network.cpp @@ -37,7 +37,6 @@ Implementation of the Network class #include #include #include -#include #include #include #include @@ -122,28 +121,6 @@ std::shared_ptr Network::addRegion(std::shared_ptr& r) { return r; } -// TODO:cereal Remove -std::shared_ptr Network::addRegionFromBundle(const std::string name, - const std::string nodeType, - const Dimensions& dimensions, - const std::string& filename, - const std::string& label) { - if (regions_.find(name) != regions_.end()) - NTA_THROW << "addRegionFromBundle; region '" - << name << "' already exists."; - if (!Path::exists(filename)) - NTA_THROW << "addRegionFromBundle; file does not exist; '" << filename << "'"; - - std::ifstream in(filename, std::ios_base::in | std::ios_base::binary); - in.exceptions(std::ifstream::failbit | std::ifstream::badbit); - std::shared_ptr r = std::make_shared(this); - r->load(in); - regions_[name] = r; - initialized_ = false; - - setDefaultPhase_(r.get()); - return r; -} void Network::setDefaultPhase_(Region *region) { UInt32 newphase = (UInt32)phaseInfo_.size(); @@ -529,156 +506,6 @@ UInt32 Network::getMaxEnabledPhase() const { return maxEnabledPhase_; } - -void Network::save(std::ostream &f) const { - // save Network, Region, Links - - f << "Network: {\n"; - f << "iteration: " << iteration_ << "\n"; - f << "Regions: " << "[ " << regions_.size() << "\n"; - - for(auto iter = regions_.cbegin(); iter != regions_.cend(); ++iter){ - std::shared_ptr r = iter->second; - r->save(f); - } - f << "]\n"; // end of regions - - // Save the Links - // determine the number of links to save. - Size count = 0; - for(auto iter = regions_.cbegin(); iter != regions_.cend(); ++iter){ - std::shared_ptr r = iter->second; - const std::map inputs = r->getInputs(); - for (const auto & inputs_input : inputs) - { - const std::vector>& links = inputs_input.second->getLinks(); - count += links.size(); - } - } - - f << "Links: [ " << count << "\n"; - - // Now serialize the links - for(auto iter = regions_.cbegin(); iter != regions_.cend(); ++iter){ - std::shared_ptr r = iter->second; - const std::map inputs = r->getInputs(); - for (const auto & inputs_input : inputs) - { - const std::vector>& links = inputs_input.second->getLinks(); - for (const auto & links_link : links) - { - auto l = links_link; - l->serialize(f); - } - - } - } - f << "]\n"; // end of links - - f << "}\n"; // end of network - f << std::endl; -} - - - - -void Network::load(std::istream &f) { - - std::string tag; - Size count; - - // Remove all existing regions and links - for (auto p: regions_) { - std::shared_ptr r = p.second; - removeRegion(r->getName()); - } - initialized_ = false; - - - f >> tag; - NTA_CHECK(tag == "Network:") << "Invalid network structure file -- does not contain 'Network' as starting tag."; - f >> tag; - NTA_CHECK(tag == "{") << "Expected beginning of a map."; - f >> tag; - NTA_CHECK(tag == "iteration:"); - f >> iteration_; - - // Regions - f >> tag; - NTA_CHECK(tag == "Regions:"); - f >> tag; - NTA_CHECK(tag == "[") << "Expected the beginning of a list"; - f >> count; - for (Size n = 0; n < count; n++) - { - std::shared_ptr r = std::make_shared(this); - r->load(f); - regions_[r->getName()] = r; - - // We must make a copy of the phases set here because - // setPhases_ will be passing this back down into - // the region. - std::set phases = r->getPhases(); - setPhases_(r.get(), phases); - - } - f >> tag; - NTA_CHECK(tag == "]") << "Expected end of list of regions."; - - - // Links - f >> tag; - NTA_CHECK(tag == "Links:"); - f >> tag; - NTA_CHECK(tag == "[") << "Expected beginning of list of links."; - f >> count; - - for (Size n=0; n < count; n++) - { - // Create the link - std::shared_ptr newLink = std::make_shared(); - newLink->deserialize(f); - - // Now connect the links to the regions - const std::string srcRegionName = newLink->getSrcRegionName(); - NTA_CHECK(regions_.find(srcRegionName) != regions_.end()) - << "Invalid network structure file -- link specifies source region '" - << srcRegionName << "' but no such region exists"; - std::shared_ptr srcRegion = getRegion(srcRegionName); - - const std::string destRegionName = newLink->getDestRegionName(); - NTA_CHECK(regions_.find(destRegionName) != regions_.end()) - << "Invalid network structure file -- link specifies destination region '" - << destRegionName << "' but no such region exists"; - std::shared_ptr destRegion = getRegion(destRegionName); - - const std::string srcOutputName = newLink->getSrcOutputName(); - Output *srcOutput = srcRegion->getOutput(srcOutputName); - NTA_CHECK(srcOutput != nullptr) << "Invalid network structure file -- link specifies source output '" - << srcOutputName << "' but no such name exists"; - - const std::string destInputName = newLink->getDestInputName(); - Input *destInput = destRegion->getInput(destInputName); - NTA_CHECK(destInput != nullptr) << "Invalid network structure file -- link specifies destination input '" - << destInputName << "' but no such name exists"; - - newLink->connectToNetwork(srcOutput, destInput); - destInput->addLink(newLink, srcOutput); - - // The Links will not be initialized. So must call net.initialize() after load(). - } // links - - - - f >> tag; - NTA_CHECK(tag == "]"); // end of links - f >> tag; - NTA_CHECK(tag == "}"); // end of network - f.ignore(1); - - post_load(); -} - void Network::post_load(std::vector>& links) { for(auto alink: links) { auto l = link( alink->getSrcRegionName(), diff --git a/src/nupic/engine/Network.hpp b/src/nupic/engine/Network.hpp index 7ddbade6ab..fbb51953e7 100644 --- a/src/nupic/engine/Network.hpp +++ b/src/nupic/engine/Network.hpp @@ -106,26 +106,23 @@ class Link; * @{ */ /** - * saveToFile(path) - * save(ostream f) - * f << net; - * serialize everything into one stream. This can be - * opened to a file or a memory stream but must be binary. + * saveToFile(path) Open a file and stream to it. (Binary) + * save(ostream f [, fmt]) Stream to your stream. + * f << net; Output human readable text. * * loadFromFile(path) - * load(istream f) - * f >> net; - * restores the streamed Network and all its parts back to - * what it was before being serialized. + * load(istream f [, fmt]) * * @path The filename into which to save/load the streamed serialization. * @f The stream with which to save/load the serialization. + * @fmt Format: One of following from enum SerializableFormat + * BINARY - A binary format which is the fastest but not portable between platforms (default). + * PORTABLE - Another Binary format, not quite as fast but is portable between platforms. + * JSON - Human readable JSON text format. Slow. + * XML - Human readable XML text format. Even slower. * - * See Serializable base class for definitions. + * See Serializable base class for more details. */ - virtual void save(std::ostream &f) const override; // TODO:cereal Remove - virtual void load(std::istream &stream) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template @@ -175,28 +172,15 @@ class Link; const std::string &nodeType, const std::string &nodeParams); - /** - * Add a region in a network from deserialized region - * - * @param Region shared_ptr - * - * @returns A pointer to the newly created Region - */ - std::shared_ptr addRegion(std::shared_ptr& region); - - - /** - * Create a new region in a network from serialized region - * The serialized file must have been created by calling SaveToFile() - * directly on the region (not on Network). It restores just this one region. - * The fields dimensions and label are not used but provided for backward - * compatability. - */ - std::shared_ptr addRegionFromBundle(const std::string name, - const std::string nodeType, - const Dimensions& dimensions, - const std::string& filename, - const std::string& label = ""); + /** + * Add a region in a network from deserialized region + * + * @param Region shared_ptr + * + * @returns A pointer to the newly created Region + */ + std::shared_ptr addRegion(std::shared_ptr& region); + /** * Removes an existing region from the network. diff --git a/src/nupic/engine/Output.cpp b/src/nupic/engine/Output.cpp index f48fff2b31..4890bca1fe 100644 --- a/src/nupic/engine/Output.cpp +++ b/src/nupic/engine/Output.cpp @@ -63,7 +63,7 @@ void Output::initialize() { size_t count = dim_.getCount(); if (data_.getType() == NTA_BasicType_SDR) { - data_.allocateBuffer(dim_); + data_.allocateBuffer(dim_.asVector()); } else { data_.allocateBuffer(count); // Zero the buffer because unitialized outputs can screw up inspectors, diff --git a/src/nupic/engine/Region.cpp b/src/nupic/engine/Region.cpp index 704b6061d6..b90aabd7f9 100644 --- a/src/nupic/engine/Region.cpp +++ b/src/nupic/engine/Region.cpp @@ -41,7 +41,6 @@ Methods related to inputs and outputs are in Region_io.cpp #include #include #include -#include #include #include @@ -308,141 +307,6 @@ void Region::setPhases(std::set &phases) { phases_ = phases; } std::set &Region::getPhases() { return phases_; } - -void Region::save(std::ostream &f) const { - f << "{\n"; - f << "name: " << name_ << "\n"; - f << "nodeType: " << type_ << "\n"; - f << "phases: [ " << phases_.size() << "\n"; - for (const auto &phases_phase : phases_) { - f << phases_phase << " "; - } - f << "]\n"; - f << "outputs: [\n"; - for(auto out: outputs_) { - f << out.first << " " << out.second->getDimensions() << "\n"; - } - f << "]\n"; - f << "inputs: [\n"; - for(auto in: inputs_) { - f << in.first << " " << in.second->getDimensions() << "\n"; - } - f << "]\n"; - f << "RegionImpl:\n"; - // Now serialize the RegionImpl plugin. - BundleIO bundle(&f); - impl_->serialize(bundle); - - f << "dim: " << getDimensions() << "\n"; - - f << "}\n"; -} - -void Region::load(std::istream &f) { - char bigbuffer[5000]; - std::string tag; - Size count; - Dimensions d; - - // Each region is a map -- extract the 5 values in the map - f >> tag; - NTA_CHECK(tag == "{") << "bad region entry (not a map)"; - - // 1. name - f >> tag; - NTA_CHECK(tag == "name:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - name_ = bigbuffer; - - // 2. nodeType - f >> tag; - NTA_CHECK(tag == "nodeType:"); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - type_ = bigbuffer; - - // 3. phases - f >> tag; - NTA_CHECK(tag == "phases:"); - f >> tag; - NTA_CHECK(tag == "[") << "Expecting a sequence of phases."; - f >> count; - phases_.clear(); - for (Size i = 0; i < count; i++) - { - UInt32 val; - f >> val; - phases_.insert(val); - } - f >> tag; - NTA_CHECK(tag == "]") << "Expected end of sequence of phases."; - - // create inputs and outputs - RegionImplFactory &factory = RegionImplFactory::getInstance(); - spec_ = factory.getSpec(type_); - createInputsAndOutputs_(); - - // 4. Restore dimensions on outputs - { - f >> tag; - NTA_CHECK(tag == "outputs:"); - f >> tag; - NTA_CHECK(tag == "[") << "Expecting a sequence of outputs."; - f >> tag; - while(!f.eof() && tag != "]") { - f >> d; - auto itr = outputs_.find(tag); - if (itr != outputs_.end()) { - itr->second->setDimensions(d); - itr->second->initialize(); - } - f >> tag; - } - NTA_CHECK(tag == "]") << "Expected end of sequence of outputs."; - } - - // 5. Restore dimensions on inputs - { - f >> tag; - NTA_CHECK(tag == "inputs:"); - f >> tag; - NTA_CHECK(tag == "[") << "Expecting a sequence of inputs."; - f >> tag; - while(!f.eof() && tag != "]") { - f >> d; - auto itr = inputs_.find(tag); - if (itr != inputs_.end()) { - itr->second->setDimensions(d); - } - f >> tag; - } - NTA_CHECK(tag == "]") << "Expected end of sequence of inputs."; - } - - // 6. impl - f >> tag; - NTA_CHECK(tag == "RegionImpl:") << "Expected beginning of RegionImpl."; - f.ignore(1); - - BundleIO bundle(&f); - impl_.reset(factory.deserializeRegionImpl(type_, bundle, this)); - - - // region level dimensions - f >> tag; - NTA_CHECK(tag == "dim:"); - f >> d; - setDimensions(d); - - initialized_ = true; - - f >> tag; - NTA_CHECK(tag == "}") << "Expected end of region. Found '" << tag << "'."; -} - - - void Region::enableProfiling() { profilingEnabled_ = true; } void Region::disableProfiling() { profilingEnabled_ = false; } @@ -742,8 +606,6 @@ std::ostream &operator<<(std::ostream &f, const Region &r) { // TODO: add region impl...maybe //f << "RegionImpl:\n"; // Now serialize the RegionImpl plugin. - //BundleIO bundle(&f); - //impl_->serialize(bundle); f << "}\n"; return f; diff --git a/src/nupic/engine/Region.hpp b/src/nupic/engine/Region.hpp index 1d01481d69..ecb0ff6968 100644 --- a/src/nupic/engine/Region.hpp +++ b/src/nupic/engine/Region.hpp @@ -39,7 +39,6 @@ // objects are returned by value. #include #include -#include #include #include #include @@ -453,10 +452,7 @@ class Region : public Serializable { // These must be implemented for serialization. - void save(std::ostream &stream) const override; - void load(std::istream &stream) override; - - CerealAdapter; // see Serializable.hpp + CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template void save_ar(Archive& ar) const { diff --git a/src/nupic/engine/RegionImpl.cpp b/src/nupic/engine/RegionImpl.cpp index eaaf3f55be..397acea9eb 100644 --- a/src/nupic/engine/RegionImpl.cpp +++ b/src/nupic/engine/RegionImpl.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include diff --git a/src/nupic/engine/RegionImpl.hpp b/src/nupic/engine/RegionImpl.hpp index 280eb2d167..08849e64b8 100644 --- a/src/nupic/engine/RegionImpl.hpp +++ b/src/nupic/engine/RegionImpl.hpp @@ -144,7 +144,6 @@ #include #include #include -#include #include namespace nupic { @@ -220,10 +219,6 @@ class RegionImpl */ // static Spec* createSpec(); - // Serialize/Deserialize state. - virtual void serialize(BundleIO &bundle) = 0; - virtual void deserialize(BundleIO &bundle) = 0; - // overridden by including the macro CerealAdapter in subclass. virtual void cereal_adapter_save(ArWrapper& a) const {}; virtual void cereal_adapter_load(ArWrapper& a) {}; diff --git a/src/nupic/engine/RegionImplFactory.cpp b/src/nupic/engine/RegionImplFactory.cpp index ce8c73f7b8..b3066ffee6 100644 --- a/src/nupic/engine/RegionImplFactory.cpp +++ b/src/nupic/engine/RegionImplFactory.cpp @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include @@ -140,19 +139,6 @@ RegionImpl *RegionImplFactory::createRegionImpl(const std::string nodeType, return impl; } -RegionImpl *RegionImplFactory::deserializeRegionImpl(const std::string nodeType, - BundleIO &bundle, - Region *region) { - // TODO:cereal Remove when Cereal is complete. - RegionImpl *impl = nullptr; - - if (regionTypeMap.find(nodeType) != regionTypeMap.end()) { - impl = regionTypeMap[nodeType]->deserializeRegionImpl(bundle, region); - } else { - NTA_THROW << "Unsupported node type '" << nodeType << "'"; - } - return impl; -} RegionImpl *RegionImplFactory::deserializeRegionImpl(const std::string nodeType, ArWrapper &wrapper, Region *region) { diff --git a/src/nupic/engine/RegionImplFactory.hpp b/src/nupic/engine/RegionImplFactory.hpp index 9e6e769042..de4beca579 100644 --- a/src/nupic/engine/RegionImplFactory.hpp +++ b/src/nupic/engine/RegionImplFactory.hpp @@ -46,7 +46,6 @@ namespace nupic { class RegionImpl; class Region; class Spec; -class BundleIO; class ValueMap; class RegisteredRegionImpl; @@ -61,9 +60,6 @@ class RegionImplFactory { RegionImpl *createRegionImpl(const std::string nodeType, const std::string nodeParams, Region *region); - // Create a RegionImpl from serialized state; caller gets ownership. TODO:cereal Remove - RegionImpl *deserializeRegionImpl(const std::string nodeType, - BundleIO &bundle, Region *region); // Create a RegionImpl from serialized state; caller gets ownership. RegionImpl *deserializeRegionImpl(const std::string nodeType, ArWrapper &wrapper, Region *region); diff --git a/src/nupic/engine/RegisteredRegionImpl.hpp b/src/nupic/engine/RegisteredRegionImpl.hpp index d1c0381aeb..06d69ffadb 100644 --- a/src/nupic/engine/RegisteredRegionImpl.hpp +++ b/src/nupic/engine/RegisteredRegionImpl.hpp @@ -67,7 +67,6 @@ namespace nupic { class Spec; - class BundleIO; // TODO:cereal Remove class ArWrapper; class RegionImpl; class Region; @@ -86,7 +85,6 @@ namespace nupic virtual RegionImpl* createRegionImpl( ValueMap& params, Region *region) = 0; - virtual RegionImpl* deserializeRegionImpl( BundleIO& params, Region *region) = 0; // TODO:cereal Remove virtual RegionImpl* deserializeRegionImpl( ArWrapper& wrapper, Region *region) = 0; virtual Spec* createSpec() = 0; diff --git a/src/nupic/engine/RegisteredRegionImplCpp.hpp b/src/nupic/engine/RegisteredRegionImplCpp.hpp index c81f3b5fed..3e78d208b8 100644 --- a/src/nupic/engine/RegisteredRegionImplCpp.hpp +++ b/src/nupic/engine/RegisteredRegionImplCpp.hpp @@ -61,7 +61,6 @@ namespace nupic { class Spec; - class BundleIO; // TODO:cereal Remove class ArWrapper; class RegionImpl; class Region; @@ -82,9 +81,6 @@ namespace nupic return new T(params, region); } - RegionImpl* deserializeRegionImpl( BundleIO& bundle, Region *region) override { // TODO:cereal Remove - return new T(bundle, region); - } RegionImpl* deserializeRegionImpl( ArWrapper& wrapper, Region *region) override { return new T(wrapper, region); } diff --git a/src/nupic/engine/Spec.cpp b/src/nupic/engine/Spec.cpp index 3036f6c931..06ab0b699c 100644 --- a/src/nupic/engine/Spec.cpp +++ b/src/nupic/engine/Spec.cpp @@ -103,6 +103,16 @@ bool InputSpec::operator==(const InputSpec &o) const { dataType == o.dataType && count == o.count && description == o.description; } +std::ostream& operator<< (std::ostream& out, const InputSpec& self) { + out << " description: " << self.description << "\n" + << " type: " << BasicType::getName(self.dataType) << "\n" + << " count: " << self.count << "\n" + << " required: " << self.required << "\n" + << " regionLevel: " << self.regionLevel << "\n" + << " isDefaultInput: " << self.isDefaultInput << "\n"; + return out; +} + OutputSpec::OutputSpec(std::string description, NTA_BasicType dataType, size_t count, @@ -116,6 +126,14 @@ bool OutputSpec::operator==(const OutputSpec &o) const { dataType == o.dataType && count == o.count && description == o.description; } +std::ostream& operator<< (std::ostream& out, const OutputSpec& self) { + out << " description: " << self.description << "\n" + << " type: " << BasicType::getName(self.dataType) << "\n" + << " count: " << self.count << "\n" + << " regionLevel: " << self.regionLevel << "\n" + << " isDefaultInput: " << self.isDefaultOutput << "\n"; + return out; +} CommandSpec::CommandSpec(std::string description) : description(std::move(description)) {} @@ -139,6 +157,21 @@ bool ParameterSpec::operator==(const ParameterSpec &o) const { description == o.description && constraints == o.constraints && defaultValue == o.defaultValue && accessMode == o.accessMode; } +std::ostream& operator<< (std::ostream& out, const ParameterSpec& self) { + out << " description: " << self.description << "\n" + << " type: " << BasicType::getName(self.dataType) << "\n" + << " count: " << self.count << "\n" + << " access: "; + switch(self.accessMode) { + case ParameterSpec::CreateAccess: out << "CreateAccess\n"; break; + case ParameterSpec::ReadOnlyAccess: out << "ReadOnlyAccess\n"; break; + case ParameterSpec::ReadWriteAccess: out << "ReadWriteAccess\n"; break; + default: "Unknown\n"; + } + if (!self.constraints.empty()) + out << " constraints" << self.constraints << "\n"; + return out; +} std::string Spec::toString() const { // TODO -- minimal information here; fill out with the rest of @@ -154,33 +187,42 @@ std::string Spec::toString() const { ss << "Parameters:" << "\n"; for (size_t i = 0; i < parameters.getCount(); ++i) { - const std::pair &item = - parameters.getByIndex(i); - ss << " " << item.first << "\n" - << " description: " << item.second.description << "\n" - << " type: " << BasicType::getName(item.second.dataType) << "\n" - << " count: " << item.second.count << "\n"; + const std::pair &item = parameters.getByIndex(i); + ss << " " << item.first << "\n"; + ss << item.second << "\n"; } ss << "Inputs:" << "\n"; for (size_t i = 0; i < inputs.getCount(); ++i) { - ss << " " << inputs.getByIndex(i).first << "\n"; + const std::pair &item = inputs.getByIndex(i); + ss << " " << item.first << "\n"; + ss << item.second << "\n"; } ss << "Outputs:" << "\n"; for (size_t i = 0; i < outputs.getCount(); ++i) { - ss << " " << outputs.getByIndex(i).first << "\n"; + const std::pair &item = outputs.getByIndex(i); + ss << " " << item.first << "\n"; + ss << item.second << "\n"; } - ss << "Commands:" - << "\n"; - for (size_t i = 0; i < commands.getCount(); ++i) { - ss << " " << commands.getByIndex(i).first << "\n"; + if (commands.getCount() > 0) { + ss << "Commands:" + << "\n"; + for (size_t i = 0; i < commands.getCount(); ++i) { + ss << " " << commands.getByIndex(i).first << ": " + << commands.getByIndex(i).second.description << "\n"; + } } - return ss.str(); } +std::ostream& operator<< (std::ostream& out, const Spec& self) { + out << self.toString(); + return out; +} + + } // namespace nupic diff --git a/src/nupic/engine/Spec.hpp b/src/nupic/engine/Spec.hpp index e24b1cd47a..26b4372442 100644 --- a/src/nupic/engine/Spec.hpp +++ b/src/nupic/engine/Spec.hpp @@ -120,7 +120,7 @@ class ParameterSpec { } std::string description; - // [open: current basic types are bytes/{u}int16/32/64, real32/64, BytePtr. Is + // [open: current basic types are Byte/{U}Int16/32/64, Real32/64. Is // this the right list? Should we have std::string, jsonstd::string?] NTA_BasicType dataType; // 1 = scalar; > 1 = array o fixed sized; 0 = array of unknown size @@ -169,6 +169,8 @@ class Spec { }; +std::ostream& operator<< (std::ostream& stream, const Spec& self); + } // namespace nupic #endif // NTA_SPEC_HPP diff --git a/src/nupic/ntypes/ArrayBase.cpp b/src/nupic/ntypes/ArrayBase.cpp index 3d46836e21..9445406885 100644 --- a/src/nupic/ntypes/ArrayBase.cpp +++ b/src/nupic/ntypes/ArrayBase.cpp @@ -327,48 +327,6 @@ bool operator==(const std::vector &lhs, const ArrayBase &rhs) { return compare_array_0_and_non0s_(rhs, lhs); } -//////////////////////////////////////////////////////////////////////////////// -// Stream Serialization (as binary) -//////////////////////////////////////////////////////////////////////////////// -void ArrayBase::save(std::ostream &outStream) const { - outStream << "[ " << count_ << " " << BasicType::getName(type_) << " "; - if (has_buffer() && type_ == NTA_BasicType_SDR) { - const sdr::SDR& sdr = getSDR(); - sdr.save(outStream); - } else { - - if (count_ > 0) { - Size size = count_ * BasicType::getSize(type_); - outStream.write(reinterpret_cast(buffer_.get()), size); - } - } - outStream << "]" << std::endl; -} -void ArrayBase::load(std::istream &inStream) { - std::string tag; - size_t count; - - NTA_CHECK(inStream.get() == '[') - << "Binary load of Array, expected starting '['."; - inStream >> count; - inStream >> tag; - type_ = BasicType::parse(tag); - if (count > 0 && type_ == NTA_BasicType_SDR) { - sdr::SDR *sdr = new sdr::SDR(); - sdr->load(inStream); - std::shared_ptr sp(reinterpret_cast(sdr)); - buffer_ = sp; - count_ = sdr->size; - } else { - allocateBuffer(count); - inStream.ignore(1); - inStream.read(buffer_.get(), count_ * BasicType::getSize(type_)); - } - NTA_CHECK(inStream.get() == ']') - << "Binary load of Array, expected ending ']'."; - inStream.ignore(1); // skip over the endl -} - //////////////////////////////////////////////////////////////////////////////// // Stream Serialization (as Ascii text character strings) // [ type count ( item item item ...) ... ] diff --git a/src/nupic/ntypes/ArrayBase.hpp b/src/nupic/ntypes/ArrayBase.hpp index 9f10e4d597..cc062a118f 100644 --- a/src/nupic/ntypes/ArrayBase.hpp +++ b/src/nupic/ntypes/ArrayBase.hpp @@ -177,10 +177,6 @@ namespace nupic /** * serialization and deserialization for an Array and ArrayBase */ - // binary representation - void save(std::ostream &outStream) const override; // TODO:Cereal- remove when Cereal is complete. - void load(std::istream &inStream) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization diff --git a/src/nupic/ntypes/BundleIO.hpp b/src/nupic/ntypes/BundleIO.hpp deleted file mode 100644 index 3cef025877..0000000000 --- a/src/nupic/ntypes/BundleIO.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* --------------------------------------------------------------------- - * Numenta Platform for Intelligent Computing (NuPIC) - * Copyright (C) 2013, Numenta, Inc. Unless you have an agreement - * with Numenta, Inc., for a separate license for this software code, the - * following terms and conditions apply: - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * - * http://numenta.org/licenses/ - * --------------------------------------------------------------------- - */ - -#ifndef NTA_BUNDLEIO_HPP -#define NTA_BUNDLEIO_HPP - -#include -#include - -namespace nupic { -class BundleIO { -public: - BundleIO(std::ostream *openStream) { - openOutputStream_ = openStream; - openInputStream_ = nullptr; - } - BundleIO(std::istream *openStream) { - openOutputStream_ = nullptr; - openInputStream_ = openStream; - } - - ~BundleIO() {} - - // return the stream. Caller should not close it. - std::ostream &getOutputStream() const { return *openOutputStream_; } - - std::istream &getInputStream() const { return *openInputStream_; } - -private: - std::ostream *openOutputStream_; - std::istream *openInputStream_; -}; - -} // namespace nupic - -#endif // NTA_BUNDLEIO_HPP diff --git a/src/nupic/ntypes/Dimensions.hpp b/src/nupic/ntypes/Dimensions.hpp index 922fc6c0ea..fcc36171a9 100644 --- a/src/nupic/ntypes/Dimensions.hpp +++ b/src/nupic/ntypes/Dimensions.hpp @@ -41,7 +41,7 @@ namespace nupic { -class Dimensions : public std::vector, public Serializable { +class Dimensions : public Serializable { public: /** * Create a new Dimensions object. @@ -51,18 +51,31 @@ class Dimensions : public std::vector, public Serializable { * Dimensions of size=1 and value [0] = 0 means "not known yet", see isDontCare() */ Dimensions() {}; - Dimensions(UInt x) { push_back(x); } - Dimensions(UInt x, UInt y) { push_back(x); push_back(y); } - Dimensions(UInt x, UInt y, UInt z) { push_back(x); push_back(y); push_back(z); } - Dimensions(const std::vector& v) : std::vector(v){}; - Dimensions(const Dimensions& d) : std::vector(d){}; + Dimensions(UInt x) { vec_.push_back(x); } + Dimensions(UInt x, UInt y) { vec_.push_back(x); vec_.push_back(y); } + Dimensions(UInt x, UInt y, UInt z) { vec_.push_back(x); vec_.push_back(y); vec_.push_back(z); } + Dimensions(const std::vector& v) { vec_ = v; }; + Dimensions(const Dimensions& d) : vec_(d.asVector()) {}; /** * @returns The count of cells in the grid which is the product of the sizes of * the dimensions. */ - size_t getCount() const { return((size() > 0) ? std::accumulate(begin(), end(), 1, std::multiplies()) : 0);} - + size_t getCount() const { return((vec_.size() > 0) ? std::accumulate(vec_.begin(),vec_.end(), 1, std::multiplies()) : 0);} + + std::vector& asVector() { return vec_; } + const std::vector& asVector() const { return vec_; } + + const UInt operator[](size_t idx) const { return vec_[idx]; } + const UInt operator[](int idx) const { return vec_[idx]; } + size_t size() const { return vec_.size(); } + bool empty() const { return vec_.empty(); } + void clear() { vec_.clear(); } + void resize(size_t i) { vec_.resize(i); } + std::vector::iterator begin() { return vec_.begin(); } + std::vector::iterator end() { return vec_.end(); } + void push_back(UInt x) { vec_.push_back(x); } + void push_front(UInt x) { vec_.insert(vec_.begin(), x); } /** * @@ -95,8 +108,8 @@ class Dimensions : public std::vector, public Serializable { * There is a function to check for each of these states. */ static const int DONTCARE = 0; - bool isUnspecified() const { return(size() == 0); } - bool isDontcare() const { return(size() == 1 && at(0) == DONTCARE); } + bool isUnspecified() const { return(vec_.size() == 0); } + bool isDontcare() const { return(vec_.size() == 1 && vec_[0] == DONTCARE); } bool isInvalid() const { return(!isDontcare() && getCount() == 0); } bool isSpecified() const { return(getCount() != 0); } @@ -106,33 +119,37 @@ class Dimensions : public std::vector, public Serializable { if (isDontcare()) return "[dontcare]"; std::stringstream ss; ss << "["; - for (size_t i = 0; i < size(); i++) { - if (i) ss << "," <(&n), sizeof(size_t)); - if (n > 0) - f.write(reinterpret_cast(&at(0)), n * sizeof(at(0))); + + CerealAdapter; + template + void save_ar(Archive& ar) const { + ar(vec_); } - void load(std::istream &f) override { - size_t n; - f.read(reinterpret_cast(&n), sizeof(size_t)); - clear(); - if (n > 0) { - resize(n); - f.read(reinterpret_cast(&at(0)), n * sizeof(at(0))); - } + template + void load_ar(Archive& ar) { + ar(vec_); } +private: + std::vector vec_; + }; + + + inline bool operator==(const Dimensions &lhs, const Dimensions &rhs) { return lhs.asVector() == rhs.asVector(); } + inline bool operator!=(const Dimensions &lhs, const Dimensions &rhs) { return !(lhs == rhs);} + + inline bool operator==(const Dimensions &lhs, const std::vector &rhs) { return lhs.asVector() == rhs; } + inline bool operator!=(const Dimensions &lhs, const std::vector &rhs) { return !(lhs == rhs);} inline std::ostream &operator<<(std::ostream &f, const Dimensions& d) { @@ -173,6 +190,7 @@ class Dimensions : public std::vector, public Serializable { } return f; } + } // namespace nupic diff --git a/src/nupic/regions/SPRegion.cpp b/src/nupic/regions/SPRegion.cpp index fb91cd3e4b..70d327ed68 100644 --- a/src/nupic/regions/SPRegion.cpp +++ b/src/nupic/regions/SPRegion.cpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -83,11 +82,6 @@ SPRegion::SPRegion(const ValueMap &values, Region *region) } -SPRegion::SPRegion(BundleIO &bundle, Region *region) - : RegionImpl(region), computeCallback_(nullptr) { - - deserialize(bundle); -} SPRegion::SPRegion(ArWrapper &wrapper, Region *region) : RegionImpl(region), computeCallback_(nullptr) { cereal_adapter_load(wrapper); @@ -920,83 +914,6 @@ void SPRegion::setParameterBool(const std::string &name, Int64 index, bool value -void SPRegion::serialize(BundleIO &bundle) { - std::ostream &f = bundle.getOutputStream(); - // There is more than one way to do this. We could serialize to YAML, which - // would make a readable format, or we could serialize directly to the stream - // Choose the fastest executing one. - f << "SPRegion " << (int)VERSION << std::endl; - f << "args " << sizeof(args_) << " "; - f.write((const char *)&args_, sizeof(args_)); - f << std::endl; - f << spatialImp_ << std::endl; - f << "outputs ["; - std::map outputs = region_->getOutputs(); - for (auto iter : outputs) { - const Array &outputBuffer = iter.second->getData(); - if (outputBuffer.getCount() != 0) { - f << iter.first << " "; - outputBuffer.save(f); - } - } - f << "] "; // end of all output buffers - - bool init = ((sp_) ? true : false); - f << init << " "; - if (init) - sp_->save(f); -} - -void SPRegion::deserialize(BundleIO &bundle) { - std::istream &f = bundle.getInputStream(); - // There is more than one way to do this. We could serialize to YAML, which - // would make a readable format, or we could serialize directly to the stream - // Choose the easier one. - char bigbuffer[5000]; - bool init; - std::string tag; - Size v; - f >> tag; - NTA_CHECK(tag == "SPRegion") - << "Bad serialization for region '" << region_->getName() - << "' of type SPRegion. Main serialization file must start " - << "with \"SPRegion\" but instead it starts with '" << tag << "'"; - - f >> v; - NTA_CHECK(v >= 1) - << "Unexpected version for SPRegion deserialization stream, " - << region_->getName(); - f >> tag; - NTA_CHECK(tag == "args"); - f >> v; - NTA_CHECK(v == sizeof(args_)); - f.ignore(1); - f.read((char *)&args_, v); - f.ignore(1); - f.getline(bigbuffer, sizeof(bigbuffer)); - spatialImp_ = bigbuffer; - f >> tag; - NTA_CHECK(tag == "outputs"); - f.ignore(1); - NTA_CHECK(f.get() == '['); // start of outputs - while (true) { - f >> tag; - f.ignore(1); - if (tag == "]") - break; - Array& a = getOutput(tag)->getData(); - a.load(f); - } - f >> init; - f.ignore(1); - if (init) { - sp_ = std::unique_ptr( - new algorithms::spatial_pooler::SpatialPooler()); - sp_->load(f); - } else - sp_ = nullptr; -} - bool SPRegion::operator==(const RegionImpl &o) const { if (o.getType() != "SPRegion") return false; SPRegion& other = (SPRegion&)o; diff --git a/src/nupic/regions/SPRegion.hpp b/src/nupic/regions/SPRegion.hpp index b79fefd126..9a3481a8e9 100644 --- a/src/nupic/regions/SPRegion.hpp +++ b/src/nupic/regions/SPRegion.hpp @@ -46,7 +46,6 @@ class SPRegion : public RegionImpl, Serializable { public: SPRegion(const ValueMap& params, Region *region); - SPRegion(BundleIO& bundle, Region* region); SPRegion(ArWrapper& wrapper, Region *region); virtual ~SPRegion(); @@ -69,10 +68,6 @@ class SPRegion : public RegionImpl, Serializable */ void initialize() override; - void serialize(BundleIO& bundle) override; - void deserialize(BundleIO& bundle) override; - - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template diff --git a/src/nupic/regions/ScalarSensor.cpp b/src/nupic/regions/ScalarSensor.cpp index c516093f8a..28792cab0e 100644 --- a/src/nupic/regions/ScalarSensor.cpp +++ b/src/nupic/regions/ScalarSensor.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include using nupic::sdr::SDR; @@ -54,10 +53,6 @@ ScalarSensor::ScalarSensor(const ValueMap ¶ms, Region *region) sensedValue_ = params.getScalarT("sensedValue"); } -ScalarSensor::ScalarSensor(BundleIO &bundle, Region *region) - : RegionImpl(region) { // TODO:cereal Remove - deserialize(bundle); -} ScalarSensor::ScalarSensor(ArWrapper &wrapper, Region *region):RegionImpl(region) { cereal_adapter_load(wrapper); } @@ -237,33 +232,6 @@ void ScalarSensor::setParameterReal64(const std::string &name, Int64 index, Real } } - - -void ScalarSensor::serialize(BundleIO &bundle) { - std::ostream &f = bundle.getOutputStream(); - f << "ScalerSensor "; - f.write((char*)¶ms_, sizeof(params_)); - f << " " << sensedValue_ << " "; - f << "~ScalerSensor" << std::endl; -} - -void ScalarSensor::deserialize(BundleIO &bundle) { - std::istream &f = bundle.getInputStream(); - std::string tag; - f >> tag; - NTA_CHECK(tag == "ScalerSensor"); - f.ignore(1); - f.read((char *)¶ms_, sizeof(params_)); - f >> sensedValue_; - f >> tag; - NTA_CHECK(tag == "~ScalerSensor"); - f.ignore(1); - - encoder_ = std::make_shared( params_ ); - - initialize(); -} - bool ScalarSensor::operator==(const RegionImpl &o) const { if (o.getType() != "ScalarSensor") return false; ScalarSensor& other = (ScalarSensor&)o; diff --git a/src/nupic/regions/ScalarSensor.hpp b/src/nupic/regions/ScalarSensor.hpp index 302e620fb7..a600104e67 100644 --- a/src/nupic/regions/ScalarSensor.hpp +++ b/src/nupic/regions/ScalarSensor.hpp @@ -48,7 +48,6 @@ namespace nupic { class ScalarSensor : public RegionImpl, Serializable { public: ScalarSensor(const ValueMap ¶ms, Region *region); - ScalarSensor(BundleIO &bundle, Region *region); // TODO:cereal Remove ScalarSensor(ArWrapper& wrapper, Region *region); virtual ~ScalarSensor() override; @@ -60,9 +59,6 @@ class ScalarSensor : public RegionImpl, Serializable { virtual void setParameterReal64(const std::string &name, Int64 index, Real64 value) override; virtual void initialize() override; - virtual void serialize(BundleIO &bundle) override; - virtual void deserialize(BundleIO &bundle) override; - void compute() override; virtual std::string executeCommand(const std::vector &args, Int64 index) override; diff --git a/src/nupic/regions/TMRegion.cpp b/src/nupic/regions/TMRegion.cpp index 0d3731e96c..e99cdf332a 100644 --- a/src/nupic/regions/TMRegion.cpp +++ b/src/nupic/regions/TMRegion.cpp @@ -78,11 +78,6 @@ TMRegion::TMRegion(const ValueMap ¶ms, Region *region) tm_ = nullptr; } -TMRegion::TMRegion(BundleIO &bundle, Region *region) - : RegionImpl(region), computeCallback_(nullptr) { - tm_ = nullptr; - deserialize(bundle); -} TMRegion::TMRegion(ArWrapper& wrapper, Region *region) : RegionImpl(region), computeCallback_(nullptr) { tm_ = nullptr; @@ -121,7 +116,7 @@ Dimensions TMRegion::askImplForOutputDimensions(const std::string &name) { // It's size is numberOfCols * args_.cellsPerColumn. // So insert a new dimension to what was provided by input. Dimensions dim = region_dim; - dim.insert(dim.begin(), args_.cellsPerColumn); + dim.push_front(args_.cellsPerColumn); return dim; } else if (name == "anomaly") { Dimensions dim{1}; @@ -178,7 +173,7 @@ void TMRegion::initialize() { nupic::algorithms::temporal_memory::TemporalMemory* tm = new nupic::algorithms::temporal_memory::TemporalMemory( - columnDimensions_, args_.cellsPerColumn, args_.activationThreshold, + columnDimensions_.asVector(), args_.cellsPerColumn, args_.activationThreshold, args_.initialPermanence, args_.connectedPermanence, args_.minThreshold, args_.maxNewSynapseCount, args_.permanenceIncrement, args_.permanenceDecrement, args_.predictedSegmentDecrement, args_.seed, args_.maxSegmentsPerCell, @@ -828,98 +823,6 @@ void TMRegion::setParameterString(const std::string &name, Int64 index, -void TMRegion::serialize(BundleIO &bundle) { - std::ostream &f = bundle.getOutputStream(); - f.precision(std::numeric_limits::digits10 + 1); - f.precision(std::numeric_limits::digits10 + 1); - - // There is more than one way to do this. We could serialize to YAML, which - // would make a readable format, or we could serialize directly to the - // stream Choose the easier one. - UInt version = VERSION; - bool init = ((tm_) ? true : false); - - f << "TMRegion " << version << std::endl; - f << init << " "; - f << sizeof(args_) << " "; - f.write((const char*)&args_, sizeof(args_)); - f << columnDimensions_ << " "; - f << std::endl; - // Need to save the output buffers - f << "outputs ["; - std::map outputs = region_->getOutputs(); - for (auto iter : outputs) { - const Array &outputBuffer = iter.second->getData(); - if (outputBuffer.getCount() != 0) { - f << iter.first << " "; - outputBuffer.save(f); - } - } - f << "] "; // end of all output buffers - if (tm_) { - tm_->save(f); - } - f << "~TMRegion "; -} - - -void TMRegion::deserialize(BundleIO &bundle) { - std::istream &f = bundle.getInputStream(); - // There is more than one way to do this. We could serialize to YAML, which - // would make a readable format, but that is a bit slow so we try to directly - // stream binary as much as we can. -// char bigbuffer[10000]; - UInt version; - Size len; - std::string tag; - bool init = false; - - f >> tag; - if (tag != "TMRegion") { - NTA_THROW << "Bad serialization for region '" << region_->getName() - << "' of type TMRegion. Main serialization file must start " - << "with \"TMRegion\" but instead it starts with '" - << tag << "'"; - } - f >> version; - NTA_CHECK(version >= VERSION) << "TMRegion deserialization, Expecting version 1 or greater."; - f >> init; - f >> len; - NTA_CHECK(len == sizeof(args_)) << "TMRegion deserialization, saved size of " - "structure args_ is wrong: " << len; - f.ignore(1); - f.read((char *)&args_, len); - f >> columnDimensions_; - - // restore output buffers - f >> tag; - NTA_CHECK(tag == "outputs"); - f.ignore(1); - NTA_CHECK(f.get() == '['); // start of outputs - while (true) { - f >> tag; - f.ignore(1); - if (tag == "]") - break; - Array& a = getOutput(tag)->getData(); - a.load(f); - } - - f >> std::ws; // ignore whitespace - - if (init) { - TemporalMemory* tm = new TemporalMemory(); - tm_.reset(tm); - - tm_->load(f); - } else { - tm_ = nullptr; - } - f >> tag; - NTA_CHECK(tag == "~TMRegion") << "expected end of TMRegion serialization"; - f.ignore(1); -} - bool TMRegion::operator==(const RegionImpl &o) const { if (o.getType() != "TMRegion") return false; TMRegion& other = (TMRegion&)o; diff --git a/src/nupic/regions/TMRegion.hpp b/src/nupic/regions/TMRegion.hpp index 979694810e..7ab996458b 100644 --- a/src/nupic/regions/TMRegion.hpp +++ b/src/nupic/regions/TMRegion.hpp @@ -46,7 +46,6 @@ class TMRegion : public RegionImpl, Serializable { TMRegion() = delete; TMRegion(const TMRegion &) = delete; TMRegion(const ValueMap ¶ms, Region *region); - TMRegion(BundleIO &bundle, Region *region); TMRegion(ArWrapper& wrapper, Region *region); virtual ~TMRegion(); @@ -67,9 +66,6 @@ class TMRegion : public RegionImpl, Serializable { */ void initialize() override; - void serialize(BundleIO &bundle) override; - void deserialize(BundleIO &bundle) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template diff --git a/src/nupic/regions/TestNode.cpp b/src/nupic/regions/TestNode.cpp index 6450217375..8999ea6877 100644 --- a/src/nupic/regions/TestNode.cpp +++ b/src/nupic/regions/TestNode.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -92,11 +91,6 @@ TestNode::TestNode(const ValueMap ¶ms, Region *region) ArWrapper arw; } -TestNode::TestNode(BundleIO &bundle, Region *region) - : RegionImpl(region), computeCallback_(nullptr), nodeCount_(1) -{ // TODO:cereal remove when Cereal is complete - deserialize(bundle); -} TestNode::TestNode(ArWrapper& wrapper, Region *region) : RegionImpl(region), computeCallback_(nullptr), nodeCount_(1) { @@ -636,113 +630,6 @@ static void arrayIn(std::istream &s, std::vector &array, } } -void TestNode::serialize(BundleIO &bundle) { - { - std::ostream &f = bundle.getOutputStream(); - // There is more than one way to do this. We could serialize to YAML, which - // would make a readable format, or we could serialize directly to the - // stream Choose the easier one. - f << "TestNode-v2" - << " " << nodeCount_ << " " << int32Param_ << " " << uint32Param_ << " " - << int64Param_ << " " << uint64Param_ << " " << real32Param_ << " " - << real64Param_ << " " << boolParam_ << " " << outputElementCount_ << " " - << delta_ << " " << iter_ << " " << dim_ << " "; - - arrayOut(f, real32ArrayParam_, "real32ArrayParam_"); - arrayOut(f, int64ArrayParam_, "int64ArrayParam_"); - arrayOut(f, boolArrayParam_, "boolArrayParam_"); - arrayOut(f, unclonedParam_, "unclonedParam_"); - f << shouldCloneParam_ << " "; - - // outer vector needs to be done by hand. - f << "unclonedArray "; - f << unclonedInt64ArrayParam_.size() << " "; // number of nodes - for (size_t i = 0; i < unclonedInt64ArrayParam_.size(); i++) { - std::stringstream name; - name << "unclonedInt64ArrayParam[" << i << "]"; - arrayOut(f, unclonedInt64ArrayParam_[i], name.str()); - } - // save the output buffers - f << "outputs ["; - std::map outputs = region_->getOutputs(); - for (auto iter : outputs) { - const Array &outputBuffer = iter.second->getData(); - if (outputBuffer.getCount() != 0) { - f << iter.first << " "; - outputBuffer.save(f); - } - } - f << "] "; // end of all output buffers - } // main file - - } - -void TestNode::deserialize(BundleIO &bundle) { - { - std::istream &f = bundle.getInputStream(); - // There is more than one way to do this. We could serialize to YAML, which - // would make a readable format, or we could serialize directly to the - // stream Choose the easier one. - std::string versionString; - f >> versionString; - if (versionString != "TestNode-v2") { - NTA_THROW << "Bad serialization for region '" << region_->getName() - << "' of type TestNode. Main serialization file must start " - << "with \"TestNode-v2\" but instead it starts with '" - << versionString << "'"; - } - f >> nodeCount_; - f >> int32Param_; - f >> uint32Param_; - f >> int64Param_; - f >> uint64Param_; - f >> real32Param_; - f >> real64Param_; - f >> boolParam_; - f >> outputElementCount_; - f >> delta_; - f >> iter_; - f >> dim_; - - arrayIn(f, real32ArrayParam_, "real32ArrayParam_"); - arrayIn(f, int64ArrayParam_, "int64ArrayParam_"); - arrayIn(f, int64ArrayParam_, "boolArrayParam_"); - arrayIn(f, unclonedParam_, "unclonedParam_"); - - f >> shouldCloneParam_; - - std::string tag; - f >> tag; - if (tag != "unclonedArray") - NTA_THROW << "Missing label for uncloned array. Got '" << tag << "'"; - size_t vecsize; - f >> vecsize; - unclonedInt64ArrayParam_.clear(); - unclonedInt64ArrayParam_.resize(vecsize); - for (size_t i = 0; i < vecsize; i++) - { - std::stringstream name; - name << "unclonedInt64ArrayParam[" << i << "]"; - arrayIn(f, unclonedInt64ArrayParam_[i], name.str()); - } - - // Restore outputs - f >> tag; - NTA_CHECK(tag == "outputs"); - f.ignore(1); - NTA_CHECK(f.get() == '['); // start of outputs - - while (true) { - f >> tag; - f.ignore(1); - if (tag == "]") - break; - getOutput(tag)->getData().load(f); - } - } - - } - bool TestNode::operator==(const RegionImpl &o) const { if (o.getType() != "TestNode") return false; diff --git a/src/nupic/regions/TestNode.hpp b/src/nupic/regions/TestNode.hpp index 28fe6c0b72..69358c7783 100644 --- a/src/nupic/regions/TestNode.hpp +++ b/src/nupic/regions/TestNode.hpp @@ -60,7 +60,6 @@ class TestNode : public RegionImpl, Serializable { public: typedef void (*computeCallbackFunc)(const std::string &); TestNode(const ValueMap ¶ms, Region *region); - TestNode(BundleIO &bundle, Region *region); // TODO:cereal Remove TestNode(ArWrapper& wrapper, Region *region); virtual ~TestNode(); @@ -79,9 +78,6 @@ class TestNode : public RegionImpl, Serializable { void initialize() override; - void serialize(BundleIO &bundle) override; - void deserialize(BundleIO &bundle) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization template diff --git a/src/nupic/regions/VectorFile.cpp b/src/nupic/regions/VectorFile.cpp index 80822b0f51..2c1909059b 100644 --- a/src/nupic/regions/VectorFile.cpp +++ b/src/nupic/regions/VectorFile.cpp @@ -348,28 +348,6 @@ void VectorFile::saveVectors(ostream &out, Size nColumns, UInt32 fileFormat, } -// For serialization -void VectorFile::save(std::ostream &f) const { - UInt32 format = (isLabeled())?1:2; // format (1 if labled, 2 if not) - f << scaleVector_.size() << " " // columns in vector - << fileVectors_.size() << " " // number of rows - << format << " "; - saveState(f); - saveVectors(f, scaleVector_.size(), format, 0, scaleVector_.size()); -} -void VectorFile::load(std::istream &f) { - size_t nCols; - size_t nRows; - int format; - f >> nCols >> nRows >> format; - - scaleVector_.resize(nCols); - offsetVector_.resize(nCols); - readState(f); - loadVectors(f, nRows, nCols, format); -} - - class AutoReleaseFile { //TODO rm this class public: FILE *file_; diff --git a/src/nupic/regions/VectorFile.hpp b/src/nupic/regions/VectorFile.hpp index 96e0c3f1dc..6deb195e44 100644 --- a/src/nupic/regions/VectorFile.hpp +++ b/src/nupic/regions/VectorFile.hpp @@ -122,9 +122,6 @@ class VectorFile : public Serializable { void saveVectors(std::ostream &out, Size nColumns, UInt32 fileFormat, Int64 begin, Int64 end, const char *lineEndings = nullptr) const; - void save(std::ostream &f) const override; - void load(std::istream &f) override; - CerealAdapter; // See Serializable.hpp template void save_ar(Archive& ar) const { diff --git a/src/nupic/regions/VectorFileEffector.cpp b/src/nupic/regions/VectorFileEffector.cpp index f49e363649..dda898bb64 100644 --- a/src/nupic/regions/VectorFileEffector.cpp +++ b/src/nupic/regions/VectorFileEffector.cpp @@ -51,10 +51,6 @@ VectorFileEffector::VectorFileEffector(const ValueMap ¶ms, Region* region) filename_ = ""; } -VectorFileEffector::VectorFileEffector(BundleIO &bundle, Region* region) - : RegionImpl(region), dataIn_(NTA_BasicType_Real32), filename_(""), - outFile_(nullptr) {} - VectorFileEffector::VectorFileEffector(ArWrapper& wrapper, Region* region) : RegionImpl(region), dataIn_(NTA_BasicType_Real32), filename_(""), outFile_(nullptr) { @@ -239,11 +235,6 @@ VectorFileEffector::getNodeOutputElementCount(const std::string &outputName) con } -void VectorFileEffector::serialize(BundleIO &bundle) { return; } - -void VectorFileEffector::deserialize(BundleIO &bundle) { return; } - - bool VectorFileEffector::operator==(const RegionImpl &o) const { if (o.getType() != "VectorFileEffector") return false; VectorFileEffector& other = (VectorFileEffector&)o; diff --git a/src/nupic/regions/VectorFileEffector.hpp b/src/nupic/regions/VectorFileEffector.hpp index d305a98f6e..c9baeb79bb 100644 --- a/src/nupic/regions/VectorFileEffector.hpp +++ b/src/nupic/regions/VectorFileEffector.hpp @@ -71,21 +71,10 @@ class VectorFileEffector : public RegionImpl, Serializable { VectorFileEffector(const ValueMap ¶ms, Region *region); - VectorFileEffector(BundleIO &bundle, Region *region); VectorFileEffector(ArWrapper& wrapper, Region *region); virtual ~VectorFileEffector(); - // --- - /// Serialize state to bundle - // --- - virtual void serialize(BundleIO &bundle) override; - - // --- - /// De-serialize state from bundle - // --- - virtual void deserialize(BundleIO &bundle) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization diff --git a/src/nupic/regions/VectorFileSensor.cpp b/src/nupic/regions/VectorFileSensor.cpp index a62cff675c..3db73e6419 100644 --- a/src/nupic/regions/VectorFileSensor.cpp +++ b/src/nupic/regions/VectorFileSensor.cpp @@ -38,7 +38,6 @@ #include #include #include -#include #include using namespace std; @@ -65,14 +64,6 @@ VectorFileSensor::VectorFileSensor(const ValueMap ¶ms, Region *region) filename_ = params.getString("inputFile"); } -VectorFileSensor::VectorFileSensor(BundleIO &bundle, Region *region) - : RegionImpl(region), repeatCount_(1), iterations_(0), curVector_(-1), - activeOutputCount_(0), hasCategoryOut_(false), hasResetOut_(false), - dataOut_(NTA_BasicType_Real32), categoryOut_(NTA_BasicType_Real32), - resetOut_(NTA_BasicType_Real32), filename_(""), scalingMode_("none"), - recentFile_("") { - deserialize(bundle); -} VectorFileSensor::VectorFileSensor(ArWrapper& wrapper, Region *region) : RegionImpl(region), repeatCount_(1), iterations_(0), curVector_(-1), activeOutputCount_(0), hasCategoryOut_(false), hasResetOut_(false), @@ -673,61 +664,6 @@ size_t VectorFileSensor::getParameterArrayCount(const std::string &name, Int64 i -void VectorFileSensor::serialize(BundleIO &bundle) { - std::ostream & f = bundle.getOutputStream(); - f << repeatCount_ << " " << activeOutputCount_ << " " << curVector_ << " " - << iterations_ << " " << hasCategoryOut_ << " " << hasResetOut_ << " " - << ((filename_ == "")?std::string("empty"):filename_) << " " - << ((scalingMode_ == "")?std::string("empty"):scalingMode_) << " " - << ((recentFile_ == "")?std::string("empty"):recentFile_) << std::endl; - f << "outputs ["; - std::map outputs = region_->getOutputs(); - for (auto iter : outputs) { - const Array &outputBuffer = iter.second->getData(); - if (outputBuffer.getCount() != 0) { - f << iter.first << " "; - outputBuffer.save(f); - } - } - f << "] "; // end of all output buffers - - f << "[" << std::endl; - vectorFile_.save(f); - f << "]" << std::endl; - f.flush(); -} - -void VectorFileSensor::deserialize(BundleIO &bundle) { - std::istream& f = bundle.getInputStream(); - std::string tag; - f >> repeatCount_ >> activeOutputCount_ >> curVector_ >> iterations_ >> - hasCategoryOut_ >> hasResetOut_; - f >> filename_ >> scalingMode_ >> recentFile_; - if (filename_ == "empty") filename_ = ""; - if (scalingMode_ == "empty") scalingMode_ = ""; - if (recentFile_ == "empty") recentFile_ = ""; - f >> tag; - NTA_CHECK(tag == "outputs"); - f.ignore(1); - NTA_CHECK(f.get() == '['); // start of outputs - while (true) { - f >> tag; - f.ignore(1); - if (tag == "]") - break; - Array& a = getOutput(tag)->getData(); - a.load(f); - } - f >> tag; - NTA_CHECK(tag == "["); - f.ignore(1); - vectorFile_.load(f); - f >> tag; - NTA_CHECK(tag == "]") << "Expected the end of vectorFile.load"; - f.ignore(1); -} - - bool VectorFileSensor::operator==(const RegionImpl &o) const { if (o.getType() != "VectorFileSensor") return false; VectorFileSensor& other = (VectorFileSensor&)o; diff --git a/src/nupic/regions/VectorFileSensor.hpp b/src/nupic/regions/VectorFileSensor.hpp index c536fbbea2..85ab1a4e8c 100644 --- a/src/nupic/regions/VectorFileSensor.hpp +++ b/src/nupic/regions/VectorFileSensor.hpp @@ -289,21 +289,10 @@ class VectorFileSensor : public RegionImpl, Serializable { VectorFileSensor(const ValueMap ¶ms, Region *region); - VectorFileSensor(BundleIO &bundle, Region *region); VectorFileSensor(ArWrapper& wrapper, Region *region); virtual ~VectorFileSensor(); - // --- - /// Serialize state to bundle - // --- - virtual void serialize(BundleIO &bundle) override; - - // --- - /// De-serialize state from bundle - // --- - virtual void deserialize(BundleIO &bundle) override; - CerealAdapter; // see Serializable.hpp // FOR Cereal Serialization diff --git a/src/nupic/types/Sdr.cpp b/src/nupic/types/Sdr.cpp index c76ad83dba..394d8f1de8 100644 --- a/src/nupic/types/Sdr.cpp +++ b/src/nupic/types/Sdr.cpp @@ -330,12 +330,12 @@ namespace sdr { void SparseDistributedRepresentation::killCells(const Real fraction, const UInt seed) { NTA_CHECK( fraction >= 0.0 ); NTA_CHECK( fraction <= 1.0 ); - const UInt nkill = round( size * fraction ); + const UInt nkill = static_cast(round( size * fraction )); Random rng(seed); auto &data = getDense(); - std::vector indices(size); - std::iota(indices.begin(), indices.end(), 0); //fills with 0,..,size-1 - const auto toKill = rng.sample(indices, nkill); // select nkill indices to be "killed", set to OFF/0 + std::vector indices(size); + std::iota(indices.begin(), indices.end(), 0); //fills with 0,..,size-1 + const auto toKill = rng.sample(indices, nkill); // select nkill indices to be "killed", set to OFF/0 for(const auto dis: toKill) { data[dis] = 0; } @@ -493,71 +493,6 @@ namespace sdr { } - void SparseDistributedRepresentation::save(std::ostream &outStream) const { - - auto writeVector = [&outStream] (const vector &vec) { - outStream << vec.size() << " "; - for( auto elem : vec ) { - outStream << elem << " "; - } - outStream << endl; - }; - - // Write a starting marker and version. - outStream << "SDR " << SERIALIZE_VERSION << " " << endl; - - // Store the dimensions. - writeVector( dimensions ); - - // Store the data in the flat-sparse format. - writeVector( getSparse() ); - - outStream << "~SDR" << endl; - } - - void SparseDistributedRepresentation::load(std::istream &inStream) { - - auto readVector = [&inStream] (vector &vec) { //TODO add to Serializable - vec.clear(); - UInt size; - inStream >> size; - vec.reserve( size ); - for( UInt i = 0; i < size; ++i ) { - UInt elem; - inStream >> elem; - vec.push_back( elem ); - } - }; - - // Read the starting marker and version. - string marker; - UInt version; - inStream >> marker >> version; - NTA_CHECK( marker == "SDR" ); - NTA_CHECK( version == SERIALIZE_VERSION ); - - // Read the dimensions. - readVector( dimensions_ ); - - // Initialize the SDR. - // Calculate the SDR's size. - size_ = 1; - for(UInt dim : dimensions) - size_ *= dim; - // Initialize sparse tuple. - coordinates_.assign( dimensions.size(), {} ); - - // Read the data. - readVector( sparse_ ); - setSparseInplace(); - - // Consume the end marker. - inStream >> marker; - NTA_CHECK( marker == "~SDR" ); - inStream.ignore(1); // skip past endl. - } - - UInt SparseDistributedRepresentation::addCallback(SDR_callback_t callback) const { UInt index = 0; for( ; index < callbacks.size(); index++ ) { diff --git a/src/nupic/types/Sdr.hpp b/src/nupic/types/Sdr.hpp index fb164afdf3..34f3131fdc 100644 --- a/src/nupic/types/Sdr.hpp +++ b/src/nupic/types/Sdr.hpp @@ -380,7 +380,7 @@ class SparseDistributedRepresentation : public Serializable NTA_ASSERT(value.size() == dimensions.size()); for(UInt dim = 0; dim < dimensions.size(); dim++) { coordinates_[dim].clear(); - coordinates_[dim].resize(value[dim].size()); + coordinates_[dim].resize(value[dim].size()); // Use an explicit type cast. Otherwise Microsoft Visual Studio will // print an excessive number of warnings. Do NOT replace this with: // coordinates_[dim].assign(value[dim].cbegin(), value[dim].cend()); @@ -563,6 +563,8 @@ class SparseDistributedRepresentation : public Serializable /** * Print a human readable version of the SDR. + * Sample output: + * "SDR( 200 ) 190, 172, 23, 118, 178, 129, 113, 71, 185, 182\n" */ friend std::ostream& operator<< (std::ostream& stream, const SparseDistributedRepresentation &sdr) { @@ -582,32 +584,56 @@ class SparseDistributedRepresentation : public Serializable } return stream << std::endl; } + friend std::istream& operator>> (std::istream& inStream, SparseDistributedRepresentation &sdr) + { + auto readVector = [&inStream] (std::vector &vec) { + inStream >> std::skipws; // skip leading whitespace + vec.clear(); + std::string val; + UInt elem = 0; + int c = 0; + while(c != '\n' && !inStream.eofbit) { + int c = inStream.get(); + if (isdigit(c)) val.append(1, c); + else if (c == ',') { + vec.push_back( atoi(val.c_str()) ); + val.clear(); + } + else if (c != ' ') break; + } + if (!val.empty()) + vec.push_back( atoi(val.c_str()) ); + }; + + // Read the starting marker and version. + std::string tag; + inStream >> tag; + NTA_CHECK( tag == "SDR(" ) << "stream does not contain an SDR from <<."; + + // Read the dimensions. + readVector( sdr.dimensions_ ); + + // Initialize the SDR. + // Calculate the SDR's size. + sdr.size_ = 1; + for(UInt dim : sdr.dimensions_) + sdr.size_ *= dim; + // Initialize sparse tuple. + sdr.coordinates_.assign( sdr.dimensions_.size(), {} ); + + // Read the data. + readVector( sdr.sparse_ ); + sdr.setSparseInplace(); + return inStream; + } bool operator==(const SparseDistributedRepresentation &sdr) const; - inline bool operator!=(const SparseDistributedRepresentation &sdr) const - { return not ((*this) == sdr); } - - -// TODO:Cereal- Remove these when Cereal is complete - /** - * Save (serialize) the current state of the SDR to the specified file. - * This method can NOT save callbacks! Only the dimensions and current data - * are saved. - * - * @param stream A valid output stream, such as an open file. - */ - void save(std::ostream &outStream) const override; + { return not ((*this) == sdr); } /** - * Load (deserialize) and initialize the SDR from the specified input - * stream. This method does NOT load callbacks! If the original SDR had - * callbacks then the user must re-add them after saving & loading the SDR. - * - * @param stream A input valid istream, such as an open file. + * Serialization routines. See Serializable.hpp */ - void load(std::istream &inStream) override; - CerealAdapter; template @@ -667,6 +693,7 @@ class SparseDistributedRepresentation : public Serializable * registered your callback. */ void removeDestroyCallback(UInt index) const; + }; typedef SparseDistributedRepresentation SDR; diff --git a/src/nupic/types/Serializable.hpp b/src/nupic/types/Serializable.hpp index 3ff410b378..e1e56a673a 100644 --- a/src/nupic/types/Serializable.hpp +++ b/src/nupic/types/Serializable.hpp @@ -168,16 +168,21 @@ typedef enum {BINARY, PORTABLE, JSON, XML} SerializableFormat; // to give names to variables in the JSON serialization. // - Serialize a raw array, start with length followed by serialization of each element. // the sizeTag starts a sequence. -// ar(cereal::make_size_tag(static_cast(count_))); -// for (size_t i = 0; i < count_; i++) +// cereal::size_type count = array_size; +// ar(cereal::make_size_tag(count)); +// for (size_t i = 0; i < static_cast(count); i++) // ar( array[i] ); -// NOTE: this has some problems. Sometimes, particularly in JSON, it will crash. -// Recommend creating a std::vector<> and then serializing the vector. -// In other words, always serialize objects, not a sequence. +// Make sure the argument to make_size_tag(count) is cereal::size_type otherwise +// things will crash in strange ways. Inside the loop there must be exactly 'count' number +// of items passed to ar( a ). Something like ar(a,b,c) is three items. If you are off +// by even one the load_ar( ) will crash on the next item because it will be the wrong +// thing in the stream. +// NOTE: If you have problems, recommend creating a std::vector<> and then serializing +// the vector. In other words, serialize objects, not a sequence. // - Serialize an std::pair // ar(cereal::make_map_item(it->first, it->second)); -// - Extra attention is needed if a variable is in a base class. See docs. -// - Extra attention may be needed for some private variables. See docs. +// - Extra attention is needed if a variable is in a base class. See Cereal docs. +// - Extra attention may be needed for some private variables. See Cereal docs. // class ArWrapper { @@ -214,25 +219,6 @@ class Serializable { Serializable() {} virtual inline int getSerializableVersion() const { return SERIALIZABLE_VERSION; } -// TODO:Cereal- To be removed after Cereal is in place. - virtual inline void saveToFile(std::string filePath) const { - std::string dirPath = Path::getParent(filePath); - Directory::create(dirPath, true, true); - std::ofstream out(filePath, std::ios_base::out | std::ios_base::binary); - out.exceptions(std::ofstream::failbit | std::ofstream::badbit); - out.precision(std::numeric_limits::digits10 + 1); - out.precision(std::numeric_limits::digits10 + 1); - save(out); - out.close(); - } - -// TODO:Cereal- To be removed after Cereal is in place. - virtual inline void loadFromFile(std::string filePath) { - std::ifstream in(filePath, std::ios_base::in | std::ios_base::binary); - in.exceptions(std::ifstream::failbit | std::ifstream::badbit); - load(in); - in.close(); - } // TODO:Cereal- after all serialization using Cereal is complete, // remove save() and load() pairs from all derived classes // change saveToStream_ar() to save() @@ -240,12 +226,9 @@ class Serializable { // change saveToFile_ar() to saveToFile(). // change loadFromFile_ar() to loadFromFile(). - // These must be implemented by the subclass. - virtual void save(std::ostream &stream) const { }; - virtual void load(std::istream &stream) { }; - virtual inline void saveToFile_ar(std::string filePath, SerializableFormat fmt=SerializableFormat::BINARY) const { + virtual inline void saveToFile(std::string filePath, SerializableFormat fmt=SerializableFormat::BINARY) const { std::string dirPath = Path::getParent(filePath); Directory::create(dirPath, true, true); std::ios_base::openmode mode = std::ios_base::out; @@ -253,12 +236,12 @@ class Serializable { std::ofstream out(filePath, mode); out.precision(std::numeric_limits::digits10 + 1); out.precision(std::numeric_limits::digits10 + 1); - saveToStream_ar(out, fmt); + save(out, fmt); out.close(); } // NOTE: for BINARY and PORTABLE the stream must be ios_base::binary or it will crash on Windows. - virtual inline void saveToStream_ar(std::ostream &out, SerializableFormat fmt=SerializableFormat::BINARY) const { + virtual inline void save(std::ostream &out, SerializableFormat fmt=SerializableFormat::BINARY) const { ArWrapper arw; arw.fmt = fmt; switch(fmt) { @@ -286,19 +269,19 @@ class Serializable { } } - virtual inline void loadFromFile_ar(std::string filePath, SerializableFormat fmt=SerializableFormat::BINARY) { + virtual inline void loadFromFile(std::string filePath, SerializableFormat fmt=SerializableFormat::BINARY) { std::ios_base::openmode mode = std::ios_base::in; if (fmt <= SerializableFormat::PORTABLE) mode |= std::ios_base::binary; std::ifstream in(filePath, mode); // NOTE: do NOT set stream exceptions: // in.exceptions(std::ifstream::failbit | std::ifstream::badbit); // The JSON parser will not be able to find the end of the parse. - loadFromStream_ar(in, fmt); + load(in, fmt); in.close(); } // NOTE: for BINARY and PORTABLE the stream must opened with ios_base::binary or it will crash on Windows. // Stream exceptions should NOT be set. - virtual inline void loadFromStream_ar(std::istream &in, SerializableFormat fmt=SerializableFormat::BINARY) { + virtual inline void load(std::istream &in, SerializableFormat fmt=SerializableFormat::BINARY) { ArWrapper arw; arw.fmt = fmt; switch(fmt) { @@ -332,9 +315,8 @@ class Serializable { // Note: if you get a compile error saying this is not defined, // or that "cannot instantiate abstract class" // add the macro 'CerealAdapter' in the derived class. - // TODO:Cereal- make these pure virtual when Cereal is included everywhere. - virtual void cereal_adapter_save(ArWrapper& a) const {}; - virtual void cereal_adapter_load(ArWrapper& a) {}; + virtual void cereal_adapter_save(ArWrapper& a) const = 0; + virtual void cereal_adapter_load(ArWrapper& a) = 0; @@ -347,19 +329,19 @@ class Serializable { #define CerealAdapter \ void cereal_adapter_save(ArWrapper& a) const override { \ switch(a.fmt) { \ - case SerializableFormat::BINARY: save_ar(*a.binary_out); break; \ - case SerializableFormat::PORTABLE: save_ar(*a.portable_out); break; \ - case SerializableFormat::JSON: save_ar(*a.json_out); break; \ - case SerializableFormat::XML: save_ar(*a.xml_out); break; \ + case SerializableFormat::BINARY: CEREAL_SAVE_FUNCTION_NAME(*a.binary_out); break; \ + case SerializableFormat::PORTABLE: CEREAL_SAVE_FUNCTION_NAME(*a.portable_out); break; \ + case SerializableFormat::JSON: CEREAL_SAVE_FUNCTION_NAME(*a.json_out); break; \ + case SerializableFormat::XML: CEREAL_SAVE_FUNCTION_NAME(*a.xml_out); break; \ default: NTA_THROW << "unknown serialization format."; \ } \ } \ void cereal_adapter_load(ArWrapper& a) override { \ switch(a.fmt) { \ - case SerializableFormat::BINARY: load_ar(*a.binary_in); break; \ - case SerializableFormat::PORTABLE: load_ar(*a.portable_in); break; \ - case SerializableFormat::JSON: load_ar(*a.json_in); break; \ - case SerializableFormat::XML: load_ar(*a.xml_in); break; \ + case SerializableFormat::BINARY: CEREAL_LOAD_FUNCTION_NAME(*a.binary_in); break; \ + case SerializableFormat::PORTABLE: CEREAL_LOAD_FUNCTION_NAME(*a.portable_in); break; \ + case SerializableFormat::JSON: CEREAL_LOAD_FUNCTION_NAME(*a.json_in); break; \ + case SerializableFormat::XML: CEREAL_LOAD_FUNCTION_NAME(*a.xml_in); break; \ default: NTA_THROW << "unknown serialization format."; \ } \ } diff --git a/src/nupic/utils/Random.hpp b/src/nupic/utils/Random.hpp index 4551ae60c9..4a64cd5959 100644 --- a/src/nupic/utils/Random.hpp +++ b/src/nupic/utils/Random.hpp @@ -78,11 +78,6 @@ class Random : public Serializable { public: Random(UInt64 seed = 0); - // save and load serialized data - void save(std::ostream &stream) const override { stream << *this; } - void load(std::istream &stream) override { stream >> *this; } - void saveToFile(std::string filePath) const override { Serializable::saveToFile(filePath); } - void loadFromFile(std::string filePath) override { Serializable::loadFromFile(filePath); } CerealAdapter; template diff --git a/src/test/unit/algorithms/AnomalyLikelihoodTest.cpp b/src/test/unit/algorithms/AnomalyLikelihoodTest.cpp index 856eae1ddf..1f852a663c 100644 --- a/src/test/unit/algorithms/AnomalyLikelihoodTest.cpp +++ b/src/test/unit/algorithms/AnomalyLikelihoodTest.cpp @@ -38,8 +38,8 @@ TEST(DISABLED_AnomalyLikelihood, SerializationLikelihood) AnomalyLikelihood b; std::stringstream ss; - a.saveToStream_ar(ss); - b.loadFromStream_ar(ss); + a.save(ss); + b.load(ss); EXPECT_EQ(a, b); } diff --git a/src/test/unit/algorithms/ConnectionsTest.cpp b/src/test/unit/algorithms/ConnectionsTest.cpp index 66e4237014..14802e2675 100644 --- a/src/test/unit/algorithms/ConnectionsTest.cpp +++ b/src/test/unit/algorithms/ConnectionsTest.cpp @@ -669,8 +669,8 @@ TEST(ConnectionsTest, testSaveLoad) { { stringstream ss; - c1.saveToStream_ar(ss); - c2.loadFromStream_ar(ss); + c1.save(ss); + c2.load(ss); } ASSERT_EQ(c1, c2); diff --git a/src/test/unit/algorithms/SDRClassifierTest.cpp b/src/test/unit/algorithms/SDRClassifierTest.cpp index 70a1dd94f2..cd1f66dd2c 100644 --- a/src/test/unit/algorithms/SDRClassifierTest.cpp +++ b/src/test/unit/algorithms/SDRClassifierTest.cpp @@ -221,9 +221,9 @@ TEST(SDRClassifierTest, SaveLoad) { // Save and load. stringstream ss; - EXPECT_NO_THROW(c1.saveToStream_ar(ss)); + EXPECT_NO_THROW(c1.save(ss)); Predictor c2; - EXPECT_NO_THROW(c2.loadFromStream_ar(ss)); + EXPECT_NO_THROW(c2.load(ss)); // Expect identical results. const auto c2_out = c2.infer( 0u, A ); diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 005fb8c8e4..15f28e18bf 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -1865,11 +1865,13 @@ TEST(SpatialPoolerTest, testSaveLoad) { setup(sp1, numInputs, numColumns); ofstream outfile; - outfile.open(filename); + outfile.open(filename, ifstream::binary); + outfile.precision(std::numeric_limits::digits10 + 1); + outfile.precision(std::numeric_limits::digits10 + 1); sp1.save(outfile); outfile.close(); - ifstream infile(filename); + ifstream infile(filename, ifstream::binary); sp2.load(infile); infile.close(); @@ -1901,6 +1903,8 @@ TEST(SpatialPoolerTest, testSerialization2) { // Save initial trained model ofstream osC("outC.stream", ofstream::binary); + osC.precision(std::numeric_limits::digits10 + 1); + osC.precision(std::numeric_limits::digits10 + 1); sp1.save(osC); osC.close(); @@ -1930,6 +1934,8 @@ TEST(SpatialPoolerTest, testSerialization2) { // Serialize ofstream os("outC.stream", ofstream::binary); + os.precision(std::numeric_limits::digits10 + 1); + os.precision(std::numeric_limits::digits10 + 1); spTemp.save(os); os.close(); @@ -1989,7 +1995,9 @@ TEST(SpatialPoolerTest, testSerialization_ar) { // Save initial trained model stringstream ss; - sp1.saveToStream_ar(ss); + ss.precision(std::numeric_limits::digits10 + 1); + ss.precision(std::numeric_limits::digits10 + 1); + sp1.save(ss); SpatialPooler sp2; @@ -2010,7 +2018,7 @@ TEST(SpatialPoolerTest, testSerialization_ar) { // Deserialize ss.seekg(0); - spTemp.loadFromStream_ar(ss); + spTemp.load(ss); // Feed new record through SDR outputC({numColumns}); @@ -2018,7 +2026,7 @@ TEST(SpatialPoolerTest, testSerialization_ar) { // Serialize ss.clear(); - spTemp.saveToStream_ar(ss); + spTemp.save(ss); testTimer.stop(); @@ -2080,15 +2088,24 @@ TEST(SpatialPoolerTest, testConstructorVsInitialize) { } TEST(SpatialPoolerTest, ExactOutput) { - string gold = - "SDR 1 " - "1 200 " - "10 190 172 23 118 178 129 113 71 185 182 " - "~SDR"; // This is all one string. - - stringstream gold_stream( gold ); + /**** TODO: fix it. + SDR copper_sdr(); + string copper = "SDR( 200 ) 190, 172, 23, 118, 178, 129, 113, 71, 185, 182\n"; + std::stringstream copper_stream( copper ); + copper_stream >> copper_sdr; + ****/ + + SDR silver_sdr({ 200 }); + SDR_sparse_t data = {190, 172, 23, 118, 178, 129, 113, 71, 185, 182}; + silver_sdr.setSparse(data); + //silver_sdr.save(std::cout, JSON); + + string gold = "{\"dimensions\": [200],\"sparse\": [190,172,23,118,178,129,113,71,185,182]}"; + std::stringstream gold_stream( gold ); SDR gold_sdr; - gold_sdr.load( gold_stream ); + gold_sdr.load( gold_stream, JSON ); + EXPECT_EQ(silver_sdr, gold_sdr); + //EXPECT_EQ(copper_sdr, gold_sdr); SDR inputs({ 1000 }); SDR columns({ 200 }); diff --git a/src/test/unit/algorithms/TemporalMemoryTest.cpp b/src/test/unit/algorithms/TemporalMemoryTest.cpp index f145418e5e..44adad9e26 100644 --- a/src/test/unit/algorithms/TemporalMemoryTest.cpp +++ b/src/test/unit/algorithms/TemporalMemoryTest.cpp @@ -1594,10 +1594,10 @@ TEST(TemporalMemoryTest, testSaveArLoadAr) { // Using Cereal Serialization stringstream ss1; - tm1.saveToStream_ar(ss1); + tm1.save(ss1); TemporalMemory tm2; - tm2.loadFromStream_ar(ss1); + tm2.load(ss1); ASSERT_TRUE(tm1 == tm2); diff --git a/src/test/unit/encoders/ScalarEncoderTest.cpp b/src/test/unit/encoders/ScalarEncoderTest.cpp index 1ec9caf3a4..4bed567227 100644 --- a/src/test/unit/encoders/ScalarEncoderTest.cpp +++ b/src/test/unit/encoders/ScalarEncoderTest.cpp @@ -196,14 +196,15 @@ TEST(ScalarEncoder, Serialization) { std::stringstream buf; for( const auto x : inputs ) { - x->save( buf ); + x->save( buf, BINARY ); } - - // cerr << "SERIALIZED:" << endl << buf.str() << endl; + + //std::cerr << "SERIALIZED:" << std::endl << buf.str() << std::endl; + buf.seekg(0); for( const auto enc1 : inputs ) { ScalarEncoder enc2; - enc2.load( buf ); + enc2.load( buf, BINARY ); const auto &p1 = enc1->parameters; const auto &p2 = enc2.parameters; diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index 49f588e5b6..f20130bd55 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -279,10 +279,10 @@ TEST(CppRegionTest, RegionSerialization) { std::shared_ptr r1 = n.addRegion("testnode", "TestNode", "{count: 2}"); std::stringstream ss; - r1->saveToStream_ar(ss); + r1->save(ss); Region r2(&n); - r2.loadFromStream_ar(ss); + r2.load(ss); EXPECT_EQ(*r1.get(), r2); } diff --git a/src/test/unit/engine/HelloRegionTest.cpp b/src/test/unit/engine/HelloRegionTest.cpp index 6d5175cbb7..0a18f5b9db 100644 --- a/src/test/unit/engine/HelloRegionTest.cpp +++ b/src/test/unit/engine/HelloRegionTest.cpp @@ -100,11 +100,11 @@ TEST(HelloRegionTest, demo) { Network net2; { std::stringstream ss; - net.saveToStream_ar(ss, SerializableFormat::JSON); + net.save(ss, SerializableFormat::JSON); if(verbose) std::cout << "Loading from stream. \n"; if(verbose) std::cout << ss.str() << std::endl; ss.seekg(0); - net2.loadFromStream_ar(ss, SerializableFormat::JSON); + net2.load(ss, SerializableFormat::JSON); } // Note: this compares the structure (regions, links, etc) not data content or state. diff --git a/src/test/unit/engine/LinkTest.cpp b/src/test/unit/engine/LinkTest.cpp index b65f93a196..b31e6177fd 100644 --- a/src/test/unit/engine/LinkTest.cpp +++ b/src/test/unit/engine/LinkTest.cpp @@ -40,7 +40,6 @@ #include #include #include -#include #include #include #include @@ -129,7 +128,6 @@ TEST(LinkTest, DelayedLink) { MyTestNode(const ValueMap ¶ms, Region *region) : TestNode(params, region) {} - MyTestNode(BundleIO &bundle, Region *region) : TestNode(bundle, region) {} // TODO:cereal Remove MyTestNode(ArWrapper &wrapper, Region *region) : TestNode(wrapper, region) {} @@ -258,7 +256,6 @@ TEST(LinkTest, DelayedLinkSerialization) { MyTestNode(const ValueMap ¶ms, Region *region) : TestNode(params, region) {} - MyTestNode(BundleIO &bundle, Region *region) : TestNode(bundle, region) {} // TODO:cereal Remove MyTestNode(ArWrapper &wrapper, Region *region) : TestNode(wrapper, region) {} @@ -488,18 +485,11 @@ class TestRegionBase : public RegionImpl, Serializable { outputElementCount_ = 1; } - TestRegionBase(BundleIO &bundle, Region *region) : RegionImpl(region) {} // TODO:cereal Remove TestRegionBase(ArWrapper &wrapper, Region *region) : RegionImpl(region) {} virtual ~TestRegionBase() {} - // Serialize state. - void serialize(BundleIO &bundle) override {} - - // De-serialize state. Must be called from deserializing constructor - void deserialize(BundleIO &bundle) override {} - bool operator==(const RegionImpl &other) const override { NTA_THROW << "equals not implemented for this test case"; @@ -527,7 +517,16 @@ class TestRegionBase : public RegionImpl, Serializable { NTA_THROW << "TestRegionBase::getOutputSize -- unknown output " << outputName; } - + // Include the required code for serialization. + CerealAdapter; + template + void save_ar(Archive & ar) const { + ar(cereal::make_nvp("outputElementCount", outputElementCount_)); + } + template + void load_ar(Archive & ar) { + ar(cereal::make_nvp("outputElementCount", outputElementCount_)); + } private: TestRegionBase(); @@ -544,8 +543,6 @@ class L2TestRegion : public TestRegionBase { L2TestRegion(const ValueMap ¶ms, Region *region) : TestRegionBase(params, region) {} - L2TestRegion(BundleIO &bundle, Region *region) - : TestRegionBase(bundle, region) {} L2TestRegion(ArWrapper &wrapper, Region *region) : TestRegionBase(wrapper, region) {} @@ -644,8 +641,6 @@ class L4TestRegion : public TestRegionBase { L4TestRegion(const ValueMap ¶ms, Region *region) : TestRegionBase(params, region), k_(params.getScalarT("k")) {} - L4TestRegion(BundleIO &bundle, Region *region) // TODO:cereal Remove - : TestRegionBase(bundle, region), k_(0) {} L4TestRegion(ArWrapper &wrapper, Region *region) : TestRegionBase(wrapper, region), k_(0) {} diff --git a/src/test/unit/ntypes/ArrayTest.cpp b/src/test/unit/ntypes/ArrayTest.cpp index f840c78595..c420705e28 100644 --- a/src/test/unit/ntypes/ArrayTest.cpp +++ b/src/test/unit/ntypes/ArrayTest.cpp @@ -604,7 +604,7 @@ TEST_F(ArrayTest, testArrayBasefunctions) { // Only SDR has dimensions std::vector d({ 10u, 10u }); Dimensions dim(d); - sdr::SDR sdr(dim); + sdr::SDR sdr(dim.asVector()); Array s(sdr); // makes a copy of sdr Dimensions dim_s(s.getSDR().dimensions); EXPECT_EQ(dim_s, dim); diff --git a/src/test/unit/regions/SPRegionTest.cpp b/src/test/unit/regions/SPRegionTest.cpp index c1516f4fa0..3fa099e8fc 100644 --- a/src/test/unit/regions/SPRegionTest.cpp +++ b/src/test/unit/regions/SPRegionTest.cpp @@ -331,11 +331,11 @@ TEST(SPRegionTest, testSerialization) Directory::removeTree("TestOutputDir", true); VERBOSE << "Writing stream to " << Path::makeAbsolute("TestOutputDir/spRegionTest.stream") << "\n"; - net1.saveToFile_ar("TestOutputDir/spRegionTest.stream", SerializableFormat::JSON); + net1.saveToFile("TestOutputDir/spRegionTest.stream", SerializableFormat::JSON); VERBOSE << "Restore from " << Path::makeAbsolute("TestOutputDir/spRegionTest.stream") << " into a second network and compare." << std::endl; - net2.loadFromFile_ar("TestOutputDir/spRegionTest.stream", SerializableFormat::JSON); + net2.loadFromFile("TestOutputDir/spRegionTest.stream", SerializableFormat::JSON); std::shared_ptr n2region1 = net2.getRegion("region1"); std::shared_ptr n2region2 = net2.getRegion("region2"); @@ -368,10 +368,10 @@ TEST(SPRegionTest, testSerialization) parameterMap.clear(); EXPECT_TRUE(captureParameters(n2region2, parameterMap)) << "Capturing parameters before second save."; - net2.saveToFile_ar("TestOutputDir/spRegionTest.stream"); + net2.saveToFile("TestOutputDir/spRegionTest.stream"); VERBOSE << "Restore into a third network.\n"; - net3.loadFromFile_ar("TestOutputDir/spRegionTest.stream"); + net3.loadFromFile("TestOutputDir/spRegionTest.stream"); VERBOSE << "Compare changed parameters.\n"; std::shared_ptr n3region2 = net3.getRegion("region2"); EXPECT_TRUE(n3region2->getType() == "SPRegion") diff --git a/src/test/unit/regions/TMRegionTest.cpp b/src/test/unit/regions/TMRegionTest.cpp index 3d4bfd307d..3011b3ed77 100644 --- a/src/test/unit/regions/TMRegionTest.cpp +++ b/src/test/unit/regions/TMRegionTest.cpp @@ -360,11 +360,11 @@ TEST(TMRegionTest, testSerialization) { VERBOSE << "saveToFile" << std::endl; Directory::removeTree("TestOutputDir", true); - net1->saveToFile_ar("TestOutputDir/tmRegionTest.stream"); + net1->saveToFile("TestOutputDir/tmRegionTest.stream"); VERBOSE << "Restore from bundle into a second network and compare." << std::endl; net2 = new Network(); - net2->loadFromFile_ar("TestOutputDir/tmRegionTest.stream"); + net2->loadFromFile("TestOutputDir/tmRegionTest.stream"); VERBOSE << "checked restored network" << std::endl; @@ -393,11 +393,11 @@ TEST(TMRegionTest, testSerialization) { << "Capturing parameters before second save."; // serialize using a stream to a single file VERBOSE << "save second network." << std::endl; - net2->saveToFile_ar("TestOutputDir/tmRegionTest.stream"); + net2->saveToFile("TestOutputDir/tmRegionTest.stream"); VERBOSE << "Restore into a third network and compare changed parameters." << std::endl; net3 = new Network(); - net3->loadFromFile_ar("TestOutputDir/tmRegionTest.stream"); + net3->loadFromFile("TestOutputDir/tmRegionTest.stream"); std::shared_ptr n3region2 = net3->getRegion("region2"); EXPECT_TRUE(n3region2->getType() == "TMRegion") << "Failure: Restored region does not have the right type. " diff --git a/src/test/unit/regions/VectorFileTest.cpp b/src/test/unit/regions/VectorFileTest.cpp index ed903acc9d..f0c36c6fe7 100644 --- a/src/test/unit/regions/VectorFileTest.cpp +++ b/src/test/unit/regions/VectorFileTest.cpp @@ -262,10 +262,10 @@ TEST(VectorFileTest, testSerialization) net1.run(1); Directory::removeTree("TestOutputDir", true); - net1.saveToFile_ar("TestOutputDir/VectorFileTest.stream"); + net1.saveToFile("TestOutputDir/VectorFileTest.stream"); VERBOSE << "Restore into a second network and compare." << std::endl; - net3.loadFromFile_ar("TestOutputDir/VectorFileTest.stream"); + net3.loadFromFile("TestOutputDir/VectorFileTest.stream"); std::shared_ptr n3region1 = net3.getRegion("region1"); std::shared_ptr n3region3 = net3.getRegion("region3"); diff --git a/src/test/unit/types/SdrTest.cpp b/src/test/unit/types/SdrTest.cpp index ceed8ed25d..a003525a33 100644 --- a/src/test/unit/types/SdrTest.cpp +++ b/src/test/unit/types/SdrTest.cpp @@ -139,7 +139,7 @@ TEST(SdrTest, TestSDR_Examples) { TEST(SdrTest, TestReshape) { SDR A({3,4,5}); - A.randomize( 5. / A.size ); + A.randomize( 5.0f / A.size ); const auto data = A.getSparse(); // Make the SDR have only the coordinate dataformat, because that could get // corrupted by the reshape. @@ -784,8 +784,8 @@ TEST(SdrTest, TestSaveLoad) { dense.setDense(SDR_dense_t({ 0, 1, 0, 0, 1, 0, 0, 0, 1 })); stringstream ss; - dense.saveToStream_ar(ss, SerializableFormat::BINARY); - dense_2.loadFromStream_ar(ss, SerializableFormat::BINARY); + dense.save(ss, SerializableFormat::BINARY); + dense_2.load(ss, SerializableFormat::BINARY); ASSERT_TRUE( dense == dense_2 ); } diff --git a/src/test/unit/utils/RandomTest.cpp b/src/test/unit/utils/RandomTest.cpp index fcdb15079a..c7d96208c3 100644 --- a/src/test/unit/utils/RandomTest.cpp +++ b/src/test/unit/utils/RandomTest.cpp @@ -192,11 +192,11 @@ TEST(RandomTest, testSerialization_ar) { EXPECT_EQ(r1.getUInt32(), 2276275187u) << "Before serialization must be same"; // serialize std::stringstream ss; - r1.saveToStream_ar(ss); + r1.save(ss); // deserialize into r2 Random r2; - r2.loadFromStream_ar(ss); + r2.load(ss); // r1 and r2 should be identical EXPECT_EQ(r1, r2) << "load from serialization"; From dba3739666a989c1f836acb3365fc413ab7171bf Mon Sep 17 00:00:00 2001 From: David Keeney Date: Tue, 4 Jun 2019 08:52:11 -0700 Subject: [PATCH 2/9] All changes completed --- .../RandomDistributedScalarEncoder.hpp | 7 +++++ src/nupic/encoders/ScalarEncoder.hpp | 6 ++++ src/nupic/engine/Link.cpp | 30 ++++++++++++++++++- src/nupic/engine/Link.hpp | 12 ++------ src/nupic/types/Serializable.hpp | 6 ++++ src/test/unit/encoders/ScalarEncoderTest.cpp | 14 ++++----- src/test/unit/engine/LinkTest.cpp | 23 ++++++++++---- 7 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/nupic/encoders/RandomDistributedScalarEncoder.hpp b/src/nupic/encoders/RandomDistributedScalarEncoder.hpp index d96e2727f0..351249a295 100644 --- a/src/nupic/encoders/RandomDistributedScalarEncoder.hpp +++ b/src/nupic/encoders/RandomDistributedScalarEncoder.hpp @@ -134,6 +134,8 @@ class RandomDistributedScalarEncoder : public BaseEncoder ar(cereal::make_nvp("name", name)); ar(cereal::make_nvp("size", args_.size)); ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("sparsity", args_.sparsity)); + ar(cereal::make_nvp("radius", args_.radius)); ar(cereal::make_nvp("resolution", args_.resolution)); ar(cereal::make_nvp("category", args_.category)); ar(cereal::make_nvp("seed", args_.seed)); @@ -142,13 +144,18 @@ class RandomDistributedScalarEncoder : public BaseEncoder // FOR Cereal Deserialization template void load_ar(Archive& ar) { + std::vector dim; std::string name; ar(cereal::make_nvp("name", name)); + NTA_CHECK(name == "RandomDistributedScalarEncoder"); ar(cereal::make_nvp("size", args_.size)); ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("sparsity", args_.sparsity)); + ar(cereal::make_nvp("radius", args_.radius)); ar(cereal::make_nvp("resolution", args_.resolution)); ar(cereal::make_nvp("category", args_.category)); ar(cereal::make_nvp("seed", args_.seed)); + BaseEncoder::initialize({ parameters.size }); } private: RDSE_Parameters args_; diff --git a/src/nupic/encoders/ScalarEncoder.hpp b/src/nupic/encoders/ScalarEncoder.hpp index 957c2536c1..a7392f9edd 100644 --- a/src/nupic/encoders/ScalarEncoder.hpp +++ b/src/nupic/encoders/ScalarEncoder.hpp @@ -150,6 +150,9 @@ namespace encoders { ar(cereal::make_nvp("periodic", args_.periodic)); ar(cereal::make_nvp("category", args_.category)); ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("sparsity", args_.sparsity)); + ar(cereal::make_nvp("size", args_.size)); + ar(cereal::make_nvp("radius", args_.radius)); ar(cereal::make_nvp("resolution", args_.resolution)); } @@ -164,6 +167,9 @@ namespace encoders { ar(cereal::make_nvp("periodic", args_.periodic)); ar(cereal::make_nvp("category", args_.category)); ar(cereal::make_nvp("activeBits", args_.activeBits)); + ar(cereal::make_nvp("sparsity", args_.sparsity)); + ar(cereal::make_nvp("size", args_.size)); + ar(cereal::make_nvp("radius", args_.radius)); ar(cereal::make_nvp("resolution", args_.resolution)); } diff --git a/src/nupic/engine/Link.cpp b/src/nupic/engine/Link.cpp index 3ae4e503a0..89c06a01dc 100644 --- a/src/nupic/engine/Link.cpp +++ b/src/nupic/engine/Link.cpp @@ -232,7 +232,35 @@ void Link::shiftBufferedData() { } } - +std::deque Link::preSerialize() const { + std::deque delay; + if (propagationDelay_ > 0) { + // we need to capture the propagationDelayBuffer_ used for propagationDelay + // Do not copy the last entry. It is the same as the output buffer. + + // The current contents of the Destination Input buffer also needs + // to be captured as if it were the top value of the propagationDelayBuffer. + // When restored, it will be copied to the dest input buffer and popped off + // before the next execution. If there is a fanIn, we only + // want to capture the amount of the input buffer contributed by + // this link. + size_t srcCount = 0; + if (src_) { + const Array& s = src_->getData(); + srcCount = s.getCount(); + } + Array a = dest_->getData().subset(destOffset_, srcCount); + delay.push_back(a); // our part of the current Dest Input buffer. + + for (auto itr = propagationDelayBuffer_.begin(); + itr != propagationDelayBuffer_.end(); itr++) { + if (itr + 1 == propagationDelayBuffer_.end()) + break; // skip the last buffer. Its the current output. + delay.push_back(*itr); + } // end for + } + return delay; +} bool Link::operator==(const Link &o) const { if (initialized_ != o.initialized_ || diff --git a/src/nupic/engine/Link.hpp b/src/nupic/engine/Link.hpp index bc75327ea3..e2deaded4f 100644 --- a/src/nupic/engine/Link.hpp +++ b/src/nupic/engine/Link.hpp @@ -473,6 +473,7 @@ class Link : public Serializable // FOR Cereal Serialization template void save_ar(Archive& ar) const { + std::deque delay = preSerialize(); ar(cereal::make_nvp("srcRegionName", srcRegionName_), cereal::make_nvp("srcOutputName", srcOutputName_), cereal::make_nvp("destRegionName", destRegionName_), @@ -480,7 +481,7 @@ class Link : public Serializable cereal::make_nvp("destOffset", destOffset_), cereal::make_nvp("is_FanIn", is_FanIn_), cereal::make_nvp("propagationDelay", propagationDelay_), - cereal::make_nvp("propagationDelayBuffer", propagationDelayBuffer_)); + cereal::make_nvp("propagationDelayBuffer", delay)); } // FOR Cereal Deserialization template @@ -506,16 +507,9 @@ class Link : public Serializable const std::string &destInputName, const size_t propagationDelay); + std::deque preSerialize() const; - // TODO: The strings with src/dest names are redundant with - // the src_ and dest_ objects. For unit testing links, - // and for deserializing networks, we need to be able to create - // a link object without a network. and for deserializing, we - // need to be able to instantiate a link before we have instantiated - // all the regions. (Maybe this isn't true? Re-evaluate when - // more infrastructure is in place). - std::string srcRegionName_; std::string destRegionName_; std::string srcOutputName_; diff --git a/src/nupic/types/Serializable.hpp b/src/nupic/types/Serializable.hpp index e1e56a673a..804cfdfc29 100644 --- a/src/nupic/types/Serializable.hpp +++ b/src/nupic/types/Serializable.hpp @@ -184,6 +184,12 @@ typedef enum {BINARY, PORTABLE, JSON, XML} SerializableFormat; // - Extra attention is needed if a variable is in a base class. See Cereal docs. // - Extra attention may be needed for some private variables. See Cereal docs. // +// NOTE: Another restruction in the use of serialization using Cereal: +// When an Archive is applied to a new stream it will parse to the end of the +// stream. If you should then apply a second Archive to the same stream it will +// not be able to parse because it is already at the end of the stream and there +// will be a read error. So, a stream can be applied only to a single Archive +// unless it is reset to the beginning of the stream with a seekg(0). class ArWrapper { public: diff --git a/src/test/unit/encoders/ScalarEncoderTest.cpp b/src/test/unit/encoders/ScalarEncoderTest.cpp index 4bed567227..b6f7985acc 100644 --- a/src/test/unit/encoders/ScalarEncoderTest.cpp +++ b/src/test/unit/encoders/ScalarEncoderTest.cpp @@ -194,17 +194,15 @@ TEST(ScalarEncoder, Serialization) { q.sparsity = 0.15f; inputs.push_back( new ScalarEncoder( q ) ); - std::stringstream buf; - for( const auto x : inputs ) { - x->save( buf, BINARY ); - } + for( const auto enc1 : inputs ) { + std::stringstream buf; + enc1->save( buf, JSON ); - //std::cerr << "SERIALIZED:" << std::endl << buf.str() << std::endl; - buf.seekg(0); + std::cerr << "SERIALIZED:" << std::endl << buf.str() << std::endl; + buf.seekg(0); - for( const auto enc1 : inputs ) { ScalarEncoder enc2; - enc2.load( buf, BINARY ); + enc2.load( buf, JSON ); const auto &p1 = enc1->parameters; const auto &p2 = enc2.parameters; diff --git a/src/test/unit/engine/LinkTest.cpp b/src/test/unit/engine/LinkTest.cpp index b31e6177fd..e428016d66 100644 --- a/src/test/unit/engine/LinkTest.cpp +++ b/src/test/unit/engine/LinkTest.cpp @@ -370,8 +370,8 @@ TEST(LinkTest, DelayedLinkSerialization) { } // Serialize the current net - VERBOSE << "cwd=" << Directory::getCWD() << std::endl; - net.saveToFile("TestOutputDir/DelayedLinkSerialization.stream"); + VERBOSE << " cwd=" << Directory::getCWD() << std::endl; + net.saveToFile("TestOutputDir/DelayedLinkSerialization.stream", JSON); { // Output values should still be all 100's // they were not modified by the save operation. @@ -383,14 +383,16 @@ TEST(LinkTest, DelayedLinkSerialization) { } // What is serialized in the Delay buffer should be - // all 0's for first row and all 10's for the second. + // all 0's from the input buffer for first row and original + // top row of buffer (all 10's) for the second. // The stored output buffer would be all 100's. - // When restored, the first row (all 0's) will be moved to destination input buffer - // and the source output buffer (all 100's) would be rolled into bottom of delay buffer. + // When restored, the first row (all 0's) will be moved to destination input buffer. + // The last row of Delay buffer (all 10's) will roll to the first row of the Delay buffer. + // The source output buffer (all 100's) would be rolled into bottom of delay buffer. // De-serialize into a new net2 Network net2; - net2.loadFromFile("TestOutputDir/DelayedLinkSerialization.stream"); + net2.loadFromFile("TestOutputDir/DelayedLinkSerialization.stream", JSON); net2.initialize(); auto n2region1 = net2.getRegion("region1"); @@ -400,6 +402,15 @@ TEST(LinkTest, DelayedLinkSerialization) { Input *n2in2 = n2region2->getInput("bottomUpIn"); Output *n2out1 = n2region1->getOutput("bottomUpOut"); + VERBOSE << "network1\n"; + VERBOSE << " in1 buffer =" << in1->getData() << std::endl; + VERBOSE << " out1 buffer =" << out1->getData() << std::endl; + VERBOSE << " in2 buffer =" << in2->getData() << std::endl; + VERBOSE << "\nnetwork2\n"; + VERBOSE << " n2in1 buffer=" << n2in1->getData() << std::endl; + VERBOSE << " n2out1 buffer=" << n2out1->getData() << std::endl; + VERBOSE << " n2in2 buffer=" << n2in2->getData() << std::endl; + // Make sure that the buffers in the restored network look exactly like the original. ASSERT_TRUE(n2in1->getData() == in1->getData()) << "Deserialized bottomUpIn region1 input buffer does not match"; ASSERT_TRUE(n2in2->getData() == in2->getData()) << "Deserialized bottomUpIn region2 does not match"; From e5f9640ecf308af503650c3554964c79f7210577 Mon Sep 17 00:00:00 2001 From: David Keeney Date: Tue, 4 Jun 2019 15:30:05 -0700 Subject: [PATCH 3/9] Cereal for PyBindRegion --- .../bindings/algorithms/py_TemporalMemory.cpp | 4 +- bindings/py/cpp_src/plugin/PyBindRegion.cpp | 145 +++++++----------- bindings/py/cpp_src/plugin/PyBindRegion.hpp | 31 ++-- .../cpp_src/plugin/RegisteredRegionImplPy.hpp | 20 --- src/nupic/types/Sdr.hpp | 1 - 5 files changed, 77 insertions(+), 124 deletions(-) diff --git a/bindings/py/cpp_src/bindings/algorithms/py_TemporalMemory.cpp b/bindings/py/cpp_src/bindings/algorithms/py_TemporalMemory.cpp index 8a86081f38..18bcf71ddf 100644 --- a/bindings/py/cpp_src/bindings/algorithms/py_TemporalMemory.cpp +++ b/bindings/py/cpp_src/bindings/algorithms/py_TemporalMemory.cpp @@ -229,7 +229,7 @@ Resets sequence state of the TM.)"); py_HTM.def("getActiveCells", [](const HTM_t& self) { auto dims = self.getColumnDimensions(); - dims.push_back( self.getCellsPerColumn() ); + dims.push_back( static_cast(self.getCellsPerColumn()) ); SDR *cells = new SDR( dims ); self.getActiveCells(*cells); return cells; @@ -274,7 +274,7 @@ R"()"); py_HTM.def("getWinnerCells", [](const HTM_t& self) { auto dims = self.getColumnDimensions(); - dims.push_back( self.getCellsPerColumn() ); + dims.push_back( static_cast(self.getCellsPerColumn()) ); SDR *winnerCells = new SDR( dims ); self.getWinnerCells(*winnerCells); return winnerCells; diff --git a/bindings/py/cpp_src/plugin/PyBindRegion.cpp b/bindings/py/cpp_src/plugin/PyBindRegion.cpp index aa7059d295..8ac906f7f2 100644 --- a/bindings/py/cpp_src/plugin/PyBindRegion.cpp +++ b/bindings/py/cpp_src/plugin/PyBindRegion.cpp @@ -154,113 +154,84 @@ namespace py = pybind11; } - PyBindRegion::PyBindRegion(const char* module, BundleIO& bundle, Region * region, const char* className) + PyBindRegion::PyBindRegion(const char* module, ArWrapper& wrapper, Region * region, const char* className) : RegionImpl(region) , module_(module) , className_(className) { - deserialize(bundle); - // XXX ADD CHECK TO MAKE SURE THE TYPE MATCHES! + cereal_adapter_load(wrapper); } PyBindRegion::~PyBindRegion() { } - void PyBindRegion::serialize(BundleIO& bundle) + std::string PyBindRegion::pickleSerialize() { // 1. serialize main state using pickle // 2. call class method to serialize external state // 1. Serialize main state of the Python module - // We want this to end up in the open stream obtained from bundle. - // a. We first pickle the python into a temporary file. - // b. copy the file into our open stream. - - std::ifstream src; - std::streambuf* pbuf; - std::string tmp_pickle = "pickle.tmp"; - py::tuple args = py::make_tuple(tmp_pickle, "wb"); - auto f = py::module::import("__builtin__").attr("file")(*args); + // We want this to end up in the open stream obtained from bundle. + // a. We first pickle the python into a temporary file. + // b. copy the file into our open stream. + + std::string tmp_pickle = "pickle.tmp"; + py::tuple args = py::make_tuple(tmp_pickle, "wb"); + auto f = py::module::import("__builtin__").attr("file")(*args); #if PY_MAJOR_VERSION >= 3 - auto pickle = py::module::import("pickle"); + auto pickle = py::module::import("pickle"); #else - auto pickle = py::module::import("cPickle"); + auto pickle = py::module::import("cPickle"); #endif - args = py::make_tuple(node_, f, 2); // use type 2 protocol - pickle.attr("dump")(*args); - pickle.attr("close")(); - - // get the out stream - std::ostream & out = bundle.getOutputStream(); - - // copy the pickle into the out stream - src.open(tmp_pickle.c_str(), std::ios::binary); - pbuf = src.rdbuf(); - size_t size = pbuf->pubseekoff(0, src.end, src.in); - out << "Pickle " << size << std::endl; - pbuf->pubseekpos(0, src.in); - out << pbuf; - src.close(); - out << "endPickle" << std::endl; - Path::remove(tmp_pickle); + args = py::make_tuple(node_, f, 2); // use type 2 protocol + pickle.attr("dump")(*args); + pickle.attr("close")(); + + // copy the pickle into the out string + std::ifstream pfile(tmp_pickle.c_str(), std::ios::binary); + std::string content((std::istreambuf_iterator(pfile)), + std::istreambuf_iterator()); + pfile.close(); + Path::remove(tmp_pickle); + return content; + } + std::string PyBindRegion::extraSerialize() + { + std::string tmp_extra = "extra.tmp"; // 2. External state // Call the Python serializeExtraData() method to write additional data. - args = py::make_tuple(tmp_pickle); + args = py::make_tuple(tmp_extra); // Need to put the None result in py::Ptr to decrement the ref count node_.attr("serializeExtraData")(*args); - // copy the extra data into the out stream - src.open(tmp_pickle.c_str(), std::ios::binary); - pbuf = src.rdbuf(); - size = pbuf->pubseekoff(0, src.end, src.in); - out << "ExtraData " << size << std::endl; - pbuf->pubseekpos(0, src.in); - out << pbuf; - src.close(); - Path::remove(tmp_pickle); - out << "endExtraData" << std::endl; - Path::remove(tmp_pickle); + // copy the extra data into the extra string + std::ifstream efile(tmp_pickle.c_str(), std::ios::binary); + std::string extra((std::istreambuf_iterator(pfile)), + std::istreambuf_iterator()); + efile.close(); + Path::remove(tmp_extra); + return extra; } - void PyBindRegion::deserialize(BundleIO& bundle) - { + void PyBindRegion::pickleDeserialize(std::string p) { // 1. deserialize main state using pickle // 2. call class method to deserialize external state - std::ofstream des; - std::streambuf *pbuf; - std::string tmp_pickle = "pickle.tmp"; - - // get the input stream - std::string tag; - size_t size; - char buf[10000]; - std::istream & in = bundle.getInputStream(); - in >> tag; - NTA_CHECK(tag == "Pickle") << "Deserialize error, expecting start of Pickle"; - in >> size; - in.ignore(1); - pbuf = in.rdbuf(); - - // write the pickle part to pickle.tmp - des.open(tmp_pickle.c_str(), std::ios::binary); - while(size > 0) { - size_t len = (size >= sizeof(buf))?sizeof(buf): size; - pbuf->sgetn(buf, len); - des.write(buf, len); - size -= sizeof(buf); - } - des.close(); - in >> tag; - NTA_CHECK(tag == "endPickle") << "Deserialize error, expected 'endPickle'\n"; - in.ignore(1); + std::ofstream des; + std::string tmp_pickle = "pickle.tmp"; + + + std::ofstream pfile(tmp_pickle.c_str(), std::ios::binary); + pfile.write(p.c_str(), p.size()); + pfile.close(); + // Tell Python to un-pickle using what is now in the pickle.tmp file. py::args args = py::make_tuple(tmp_pickle, "rb"); @@ -276,30 +247,20 @@ namespace py = pybind11; pickle.attr("load")(*args); pickle.attr("close")(); - Path::remove(tmp_pickle); + Path::remove(tmp_pickle); + } + void PyBindRegion::extraDeserialize(std::string e) { // 2. External state - // fetch the extraData - in >> tag; - NTA_CHECK(tag == "ExtraData") << "Deserialize error, expected start of ExtraData\n"; - in >> size; - in.ignore(1); - des.open(tmp_pickle.c_str(), std::ios::binary); - while(size > 0) { - size_t len = (size >= sizeof(buf))?sizeof(buf): size; - pbuf->sgetn(buf, len); - des.write(buf, len); - size -= sizeof(buf); - } - des.close(); - in >> tag; - NTA_CHECK(tag == "endExtraData") << "Deserialize error, expected 'endExtraData'\n"; - in.ignore(1); + std::string tmp_extra = "extra.tmp"; + std::ofstream efile(tmp_extra.c_str(), std::ios::binary); + efile.write(e.c_str(), e.size()); + efile.close(); // Call the Python deSerializeExtraData() method - args = py::make_tuple(tmp_pickle); + args = py::make_tuple(tmp_extra); node_.attr("deSerializeExtraData")(*args); - Path::remove(tmp_pickle); + Path::remove(tmp_extra); } template diff --git a/bindings/py/cpp_src/plugin/PyBindRegion.hpp b/bindings/py/cpp_src/plugin/PyBindRegion.hpp index 246146a58a..9c30492323 100644 --- a/bindings/py/cpp_src/plugin/PyBindRegion.hpp +++ b/bindings/py/cpp_src/plugin/PyBindRegion.hpp @@ -57,10 +57,7 @@ namespace nupic // Constructors PyBindRegion() = delete; PyBindRegion(const char* module, const ValueMap& nodeParams, Region *region, const char* className); - PyBindRegion(const char* module, BundleIO& bundle, Region* region, const char* className); - PyBindRegion(const char* module, ArWrapper& wrapper, Region *region, const char* className) : RegionImpl(region) { - // TODO:cereal complete. - } + PyBindRegion(const char* module, ArWrapper& wrapper, Region *region, const char* className) : RegionImpl(region); // no copy constructor PyBindRegion(const Region &) = delete; @@ -69,10 +66,22 @@ namespace nupic virtual ~PyBindRegion(); - // Manual serialization methods. Current recommended method. - void serialize(BundleIO& bundle) override; - void deserialize(BundleIO& bundle) override; - + CerealAdapter; // see Serializable.hpp + // FOR Cereal Serialization + template + void save_ar(Archive& ar) const { + std::string p = pickleSerialize(); + std::string e = extraSerialize(); + ar(p, e); + } + template + void load_ar(Archive& ar) { + std::string p; + std::string e; + ar(p, e); + pickleDeserialize(p); + extraDeseerialize(e); + } bool operator==(const RegionImpl &other) const override { NTA_THROW << " == not implemented yet for PyBindRegion."; @@ -136,7 +145,11 @@ namespace nupic Spec nodeSpec_; // locally cached version of spec. - }; + std::string pickleSerialize(); + std::string extraSerialize(); + void pickleDeserialize(std::string p); + void extraDeserialize(std::string e); + }; diff --git a/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp b/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp index 08d5a4370f..9c23936643 100644 --- a/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp +++ b/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp @@ -82,7 +82,6 @@ static int python_node_count = 0; namespace nupic { class Spec; - class BundleIO; class PyRegionImpl; class Region; class ValueMap; @@ -119,25 +118,6 @@ namespace nupic } } - // use PyBindRegion class to instantiate and deserialize the python class in the specified module. TODO:cereal Remove - RegionImpl* deserializeRegionImpl(BundleIO& bundle, Region *region) override - { - try { - return new PyBindRegion(module_.c_str(), bundle, region, classname_.c_str()); - } - catch (const py::error_already_set& e) - { - throw Exception(__FILE__, __LINE__, e.what()); - } - catch (nupic::Exception & e) - { - throw nupic::Exception(e); - } - catch (...) - { - NTA_THROW << "Something bad happed while deserializing a .py region"; - } - } // use PyBindRegion class to instantiate and deserialize the python class in the specified module. RegionImpl* deserializeRegionImpl(ArWrapper& wrapper, Region *region) override { diff --git a/src/nupic/types/Sdr.hpp b/src/nupic/types/Sdr.hpp index 34f3131fdc..c5e2aed99c 100644 --- a/src/nupic/types/Sdr.hpp +++ b/src/nupic/types/Sdr.hpp @@ -590,7 +590,6 @@ class SparseDistributedRepresentation : public Serializable inStream >> std::skipws; // skip leading whitespace vec.clear(); std::string val; - UInt elem = 0; int c = 0; while(c != '\n' && !inStream.eofbit) { int c = inStream.get(); From fa399204d1d5fc4832ac735e2721bd865f89f0eb Mon Sep 17 00:00:00 2001 From: David Keeney Date: Tue, 4 Jun 2019 17:10:42 -0700 Subject: [PATCH 4/9] fixes to PyBindRegion --- bindings/py/cpp_src/bindings/engine/py_Engine.cpp | 11 ++++------- bindings/py/cpp_src/plugin/PyBindRegion.cpp | 15 +++++++-------- bindings/py/cpp_src/plugin/PyBindRegion.hpp | 8 ++++---- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/bindings/py/cpp_src/bindings/engine/py_Engine.cpp b/bindings/py/cpp_src/bindings/engine/py_Engine.cpp index 13330a430e..ef014f8593 100644 --- a/bindings/py/cpp_src/bindings/engine/py_Engine.cpp +++ b/bindings/py/cpp_src/bindings/engine/py_Engine.cpp @@ -265,13 +265,10 @@ namespace nupic_ext py_Network.def("initialize", &nupic::Network::initialize); - py_Network.def("addRegionFromBundle", &nupic::Network::addRegionFromBundle - , "A function to load a serialized region into a Network framework." - , py::arg("name") - , py::arg("nodeType") - , py::arg("dimensions") - , py::arg("filename") - , py::arg("label") = ""); + py_Network.def("save", &nupic::Network::save) + .def("load", &nupic::Network::load) + .def("saveToFile", &nupic::Network::saveToFile) + .def("loadFromFile", &nupic::Network::loadFromFile); py_Network.def("link", &nupic::Network::link , "Defines a link between regions" diff --git a/bindings/py/cpp_src/plugin/PyBindRegion.cpp b/bindings/py/cpp_src/plugin/PyBindRegion.cpp index 8ac906f7f2..b6f20dd4a6 100644 --- a/bindings/py/cpp_src/plugin/PyBindRegion.cpp +++ b/bindings/py/cpp_src/plugin/PyBindRegion.cpp @@ -38,7 +38,6 @@ In this case, the C++ engine is actually calling into the Python code. #include #include #include -#include #include #include @@ -168,7 +167,7 @@ namespace py = pybind11; { } - std::string PyBindRegion::pickleSerialize() + std::string PyBindRegion::pickleSerialize() const { // 1. serialize main state using pickle // 2. call class method to serialize external state @@ -199,21 +198,21 @@ namespace py = pybind11; Path::remove(tmp_pickle); return content; } - std::string PyBindRegion::extraSerialize() + std::string PyBindRegion::extraSerialize() const { std::string tmp_extra = "extra.tmp"; // 2. External state // Call the Python serializeExtraData() method to write additional data. - args = py::make_tuple(tmp_extra); + py::tuple args = py::make_tuple(tmp_extra); // Need to put the None result in py::Ptr to decrement the ref count node_.attr("serializeExtraData")(*args); // copy the extra data into the extra string - std::ifstream efile(tmp_pickle.c_str(), std::ios::binary); - std::string extra((std::istreambuf_iterator(pfile)), - std::istreambuf_iterator()); + std::ifstream efile(tmp_extra.c_str(), std::ios::binary); + std::string extra((std::istreambuf_iterator(efile)), + std::istreambuf_iterator()); efile.close(); Path::remove(tmp_extra); return extra; @@ -258,7 +257,7 @@ namespace py = pybind11; efile.close(); // Call the Python deSerializeExtraData() method - args = py::make_tuple(tmp_extra); + py::tuple args = py::make_tuple(tmp_extra); node_.attr("deSerializeExtraData")(*args); Path::remove(tmp_extra); } diff --git a/bindings/py/cpp_src/plugin/PyBindRegion.hpp b/bindings/py/cpp_src/plugin/PyBindRegion.hpp index 9c30492323..d5bafca24c 100644 --- a/bindings/py/cpp_src/plugin/PyBindRegion.hpp +++ b/bindings/py/cpp_src/plugin/PyBindRegion.hpp @@ -57,7 +57,7 @@ namespace nupic // Constructors PyBindRegion() = delete; PyBindRegion(const char* module, const ValueMap& nodeParams, Region *region, const char* className); - PyBindRegion(const char* module, ArWrapper& wrapper, Region *region, const char* className) : RegionImpl(region); + PyBindRegion(const char* module, ArWrapper& wrapper, Region *region, const char* className); // no copy constructor PyBindRegion(const Region &) = delete; @@ -80,7 +80,7 @@ namespace nupic std::string e; ar(p, e); pickleDeserialize(p); - extraDeseerialize(e); + extraDeserialize(e); } bool operator==(const RegionImpl &other) const override { @@ -145,8 +145,8 @@ namespace nupic Spec nodeSpec_; // locally cached version of spec. - std::string pickleSerialize(); - std::string extraSerialize(); + std::string pickleSerialize() const; + std::string extraSerialize() const; void pickleDeserialize(std::string p); void extraDeserialize(std::string e); }; From b4c8b9468417e48bfe236e22a5751574b7f04fcc Mon Sep 17 00:00:00 2001 From: David Keeney Date: Tue, 4 Jun 2019 17:36:33 -0700 Subject: [PATCH 5/9] fixed errors found by CI --- src/nupic/algorithms/SpatialPooler.hpp | 1 + src/nupic/algorithms/TemporalMemory.hpp | 2 ++ src/nupic/encoders/ScalarEncoder.cpp | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nupic/algorithms/SpatialPooler.hpp b/src/nupic/algorithms/SpatialPooler.hpp index 17150c0e76..72a26941d6 100644 --- a/src/nupic/algorithms/SpatialPooler.hpp +++ b/src/nupic/algorithms/SpatialPooler.hpp @@ -1251,6 +1251,7 @@ class SpatialPooler : public Serializable const connections::Connections &connections = connections_; }; +std::ostream & operator<<(std::ostream & out, const SpatialPooler &sp); } // end namespace spatial_pooler diff --git a/src/nupic/algorithms/TemporalMemory.hpp b/src/nupic/algorithms/TemporalMemory.hpp index 6518bb5c7a..680bacaf5f 100644 --- a/src/nupic/algorithms/TemporalMemory.hpp +++ b/src/nupic/algorithms/TemporalMemory.hpp @@ -661,6 +661,8 @@ using namespace nupic::algorithms::connections; const Real &anomaly = anomaly_; }; + std::ostream& operator<< (std::ostream& stream, const TemporalMemory& self); + } // end namespace temporal_memory } // end namespace algorithms } // namespace nupic diff --git a/src/nupic/encoders/ScalarEncoder.cpp b/src/nupic/encoders/ScalarEncoder.cpp index ec9b8406b3..0611494de8 100644 --- a/src/nupic/encoders/ScalarEncoder.cpp +++ b/src/nupic/encoders/ScalarEncoder.cpp @@ -195,7 +195,7 @@ void ScalarEncoder::encode(Real64 input, SDR &output) output.setSparse( sparse ); } -std::ostream & nupic::encoders::operator<<(std::ostream & out, const ScalarEncoder &self) +std::ostream & operator<<(std::ostream & out, const ScalarEncoder &self) { out << "ScalarEncoder "; out << "minimum: " << self.parameters.minimum << ", "; From 252727225656ae607b67f5a7b64164da1fcd98f5 Mon Sep 17 00:00:00 2001 From: David Keeney Date: Wed, 5 Jun 2019 09:59:36 -0700 Subject: [PATCH 6/9] added saveToFile and loadFromFile to Random --- .../py/cpp_src/bindings/math/py_Random.cpp | 29 +++++++++++++++++-- src/nupic/ntypes/Dimensions.hpp | 4 +-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/bindings/py/cpp_src/bindings/math/py_Random.cpp b/bindings/py/cpp_src/bindings/math/py_Random.cpp index 7ec503d72d..41d9bc52f6 100644 --- a/bindings/py/cpp_src/bindings/math/py_Random.cpp +++ b/bindings/py/cpp_src/bindings/math/py_Random.cpp @@ -109,6 +109,32 @@ namespace nupic_ext { ////////////////// // serialization ///////////////// + Random.def("saveToFile", [](Random_t& self, const std::string& name, int fmt) { + nupic::SerializableFormat fmt1; + switch(fmt) { + case 0: fmt1 = nupic::SerializableFormat::BINARY; break; + case 1: fmt1 = nupic::SerializableFormat::PORTABLE; break; + case 2: fmt1 = nupic::SerializableFormat::JSON; break; + case 3: fmt1 = nupic::SerializableFormat::XML; break; + default: NTA_THROW << "unknown serialization format."; + } + self.saveToFile(name, fmt1); + }, "serialize to a File, using BINARY=0, PORTABLE=1, JSON=2, or XML=3 format.", + py::arg("name"), py::arg("fmt") = 0); + + Random.def("loadFromFile", [](Random_t& self, const std::string& name, int fmt) { + nupic::SerializableFormat fmt1; + switch(fmt) { + case 0: fmt1 = nupic::SerializableFormat::BINARY; break; + case 1: fmt1 = nupic::SerializableFormat::PORTABLE; break; + case 2: fmt1 = nupic::SerializableFormat::JSON; break; + case 3: fmt1 = nupic::SerializableFormat::XML; break; + default: NTA_THROW << "unknown serialization format."; + } + self.loadFromFile(name, fmt1); + }, "load from a File, using BINARY, PORTABLE, JSON, or XML format.", + py::arg("name"), py::arg("fmt") = 0); + Random.def(py::pickle( [](const Random_t& r) { @@ -131,9 +157,6 @@ namespace nupic_ext { } )); - Random.def("saveToFile", &Random_t::saveToFile); - Random.def("loadFromFile", &Random_t::loadFromFile); - } } // namespace nupic_ext diff --git a/src/nupic/ntypes/Dimensions.hpp b/src/nupic/ntypes/Dimensions.hpp index fcc36171a9..4ae98c4fa9 100644 --- a/src/nupic/ntypes/Dimensions.hpp +++ b/src/nupic/ntypes/Dimensions.hpp @@ -66,8 +66,8 @@ class Dimensions : public Serializable { std::vector& asVector() { return vec_; } const std::vector& asVector() const { return vec_; } - const UInt operator[](size_t idx) const { return vec_[idx]; } - const UInt operator[](int idx) const { return vec_[idx]; } + UInt operator[](size_t idx) const { return vec_[idx]; } + UInt operator[](int idx) const { return vec_[idx]; } size_t size() const { return vec_.size(); } bool empty() const { return vec_.empty(); } void clear() { vec_.clear(); } From 418f51cf49ba928ec8f2487773f6f1ef8c13735d Mon Sep 17 00:00:00 2001 From: David Keeney Date: Fri, 7 Jun 2019 16:40:04 -0700 Subject: [PATCH 7/9] addresses review points. --- src/nupic/algorithms/SpatialPooler.cpp | 70 +++++++++++-------- src/nupic/algorithms/TemporalMemory.cpp | 63 ----------------- src/nupic/algorithms/TemporalMemory.hpp | 3 - .../RandomDistributedScalarEncoder.cpp | 10 +-- src/nupic/engine/Spec.cpp | 2 +- src/nupic/types/Sdr.hpp | 41 ----------- .../unit/algorithms/SpatialPoolerTest.cpp | 18 ++--- src/test/unit/engine/YAMLUtilsTest.cpp | 4 +- 8 files changed, 53 insertions(+), 158 deletions(-) diff --git a/src/nupic/algorithms/SpatialPooler.cpp b/src/nupic/algorithms/SpatialPooler.cpp index e775524fdb..e6e747e252 100644 --- a/src/nupic/algorithms/SpatialPooler.cpp +++ b/src/nupic/algorithms/SpatialPooler.cpp @@ -1022,40 +1022,48 @@ void SpatialPooler::printState(const vector &state, std::ostream& out) con out << "]\n"; } -std::ostream & nupic::algorithms::spatial_pooler::operator<<(std::ostream & out, const SpatialPooler &sp) -{ - sp.printParameters(out); - out << "inputDimensions: "; - sp.printState(sp.getInputDimensions(), out); - out << "columnDimensions: "; - sp.printState(sp.getColumnDimensions(), out); - out << "boostFactors: "; - sp.printState(sp.boostFactors_, out); - out << "overlapDutyCycles: "; - sp.printState(sp.overlapDutyCycles_, out); - out << "activeDutyCycles: "; - sp.printState(sp.activeDutyCycles_, out); - out << "minOverlapDutyCycles: "; - sp.printState(sp.minOverlapDutyCycles_, out); - out << "tieBreaker: "; - sp.printState(sp.tieBreaker_, out); - out << "Connections: " << sp.connections_; - return out; -} /** equals implementation based on text serialization */ bool SpatialPooler::operator==(const SpatialPooler& o) const{ - stringstream s; - s.flags(ios::scientific); - s.precision(numeric_limits::digits10 + 1); - - s << *this; - const string thisStr = s.str(); - - s.str(""); //clear stream - s << o; - const string otherStr = s.str(); + // Store the simple variables first. + if (numInputs_ != o.numInputs_) return false; + if (numColumns_ != o.numColumns_) return false; + if (potentialRadius_ != o.potentialRadius_) return false; + if (potentialPct_ != o.potentialPct_) return false; + if (initConnectedPct_ != o.initConnectedPct_) return false; + if (globalInhibition_ != o.globalInhibition_) return false; + if (numActiveColumnsPerInhArea_ != o.numActiveColumnsPerInhArea_) return false; + if (localAreaDensity_ != o.localAreaDensity_) return false; + if (stimulusThreshold_ != o.stimulusThreshold_) return false; + if (inhibitionRadius_ != o.inhibitionRadius_) return false; + if (dutyCyclePeriod_ != o.dutyCyclePeriod_) return false; + if (boostStrength_ != o.boostStrength_) return false; + if (iterationNum_ != o.iterationNum_) return false; + if (iterationLearnNum_ != o.iterationLearnNum_) return false; + if (spVerbosity_ != o.spVerbosity_) return false; + if (updatePeriod_ != o.updatePeriod_) return false; + if (synPermInactiveDec_ != o.synPermInactiveDec_) return false; + if (synPermActiveInc_ != o.synPermActiveInc_) return false; + if (synPermBelowStimulusInc_ != o.synPermBelowStimulusInc_) return false; + if (synPermConnected_ != o.synPermConnected_) return false; + if (minPctOverlapDutyCycles_ != o.minPctOverlapDutyCycles_) return false; + if (wrapAround_ != o.wrapAround_) return false; + + // compare vectors. + if (inputDimensions_ != o.inputDimensions_) return false; + if (columnDimensions_ != o.columnDimensions_) return false; + if (boostFactors_ != o.boostFactors_) return false; + if (overlapDutyCycles_ != o.overlapDutyCycles_) return false; + if (activeDutyCycles_ != o.activeDutyCycles_) return false; + if (minOverlapDutyCycles_ != o.minOverlapDutyCycles_) return false; + if (tieBreaker_ != o.tieBreaker_) return false; + + // compare connections + if (connections_ != o.connections_) return false; + + //Random + if (rng_ != o.rng_) return false; + return true; - return thisStr == otherStr; } diff --git a/src/nupic/algorithms/TemporalMemory.cpp b/src/nupic/algorithms/TemporalMemory.cpp index 2ee60dfa9c..e15c49dbf4 100644 --- a/src/nupic/algorithms/TemporalMemory.cpp +++ b/src/nupic/algorithms/TemporalMemory.cpp @@ -846,74 +846,11 @@ void TemporalMemory::printParameters(std::ostream& out) const { << "connectedPermanence = " << getConnectedPermanence() << std::endl << "minThreshold = " << getMinThreshold() << std::endl << "maxNewSynapseCount = " << getMaxNewSynapseCount() << std::endl - << "checkInputs = " << checkInputs_ << std::endl << "permanenceIncrement = " << getPermanenceIncrement() << std::endl << "permanenceDecrement = " << getPermanenceDecrement() << std::endl << "predictedSegmentDecrement = " << getPredictedSegmentDecrement() - << "anomaly = " << anomaly_ << std::endl << std::endl << "maxSegmentsPerCell = " << getMaxSegmentsPerCell() << std::endl << "maxSynapsesPerSegment = " << getMaxSynapsesPerSegment() - << "extra = " << extra_ << std::endl - << "iteration = " << iteration_ << std::endl << std::endl; } - - -std::ostream& nupic::algorithms::temporal_memory::operator<< (std::ostream& out, const TemporalMemory& tm) { - tm.printParameters(out); - - out << "Connections: " << tm.connections << endl; - out << "rng: " << tm.rng_ << endl; - - out << "columnDimensions: [ "; - for (auto &elem : tm.columnDimensions_) { - out << elem << " "; - } - out << "]" << endl; - - out << "activeCells: [ "; - for (CellIdx cell : tm.activeCells_) { - out << cell << " "; - } - out << "]" << endl; - - out << "winnerCells: [ "; - for (CellIdx cell : tm.winnerCells_) { - out << cell << " "; - } - out << "]" << endl << endl; - - out << "segmentsValid: " << tm.segmentsValid_ << " "; - out << "activeSegments: ["; - for (Segment segment : tm.activeSegments_) { - const CellIdx cell = tm.connections.cellForSegment(segment); - const vector &segments = tm.connections.segmentsForCell(cell); - - SegmentIdx idx = (SegmentIdx)std::distance( - segments.begin(), std::find(segments.begin(), segments.end(), segment)); - - out << " [" << endl; - out << idx << ","; - out << cell << ","; - out << tm.numActiveConnectedSynapsesForSegment_[segment] << "]" << endl; - } - out << "]" << endl << endl; - - out << "matchingSegments: [" << endl; - for (Segment segment : tm.matchingSegments_) { - const CellIdx cell = tm.connections.cellForSegment(segment); - const vector &segments = tm.connections.segmentsForCell(cell); - - SegmentIdx idx = (SegmentIdx)std::distance( - segments.begin(), std::find(segments.begin(), segments.end(), segment)); - - out << " [" << endl; - out << idx << ","; - out << cell << ","; - out << tm.numActivePotentialSynapsesForSegment_[segment] << "]" << endl; - } - out << "]" << endl << endl; - return out; -} - diff --git a/src/nupic/algorithms/TemporalMemory.hpp b/src/nupic/algorithms/TemporalMemory.hpp index 24a9fbabd6..6da7d53685 100644 --- a/src/nupic/algorithms/TemporalMemory.hpp +++ b/src/nupic/algorithms/TemporalMemory.hpp @@ -573,7 +573,6 @@ using namespace nupic::algorithms::connections; virtual bool operator==(const TemporalMemory &other) const; inline bool operator!=(const TemporalMemory &other) const { return not this->operator==(other); } - friend std::ostream& operator<< (std::ostream& stream, const TemporalMemory& self); //---------------------------------------------------------------------- // Debugging helpers @@ -666,8 +665,6 @@ using namespace nupic::algorithms::connections; const Real &anomaly = anomaly_; }; - std::ostream& operator<< (std::ostream& stream, const TemporalMemory& self); - } // end namespace temporal_memory } // end namespace algorithms } // namespace nupic diff --git a/src/nupic/encoders/RandomDistributedScalarEncoder.cpp b/src/nupic/encoders/RandomDistributedScalarEncoder.cpp index 5b7e2ba658..86429622dc 100644 --- a/src/nupic/encoders/RandomDistributedScalarEncoder.cpp +++ b/src/nupic/encoders/RandomDistributedScalarEncoder.cpp @@ -130,10 +130,10 @@ void RandomDistributedScalarEncoder::encode(Real64 input, sdr::SDR &output) std::ostream & nupic::encoders::operator<<(std::ostream & out, const RandomDistributedScalarEncoder &self) { out << "RDSE "; - out << "size: " << self.parameters.size << ", "; - out << "activeBits: " << self.parameters.activeBits << ", "; - out << "resolution: " << self.parameters.resolution << ", "; - out << "category: " << self.parameters.category << ", "; - out << "seed: " << self.parameters.seed << std::endl; + out << " size: " << self.parameters.size << ",\n"; + out << " activeBits: " << self.parameters.activeBits << ",\n"; + out << " resolution: " << self.parameters.resolution << ",\n"; + out << " category: " << self.parameters.category << ",\n"; + out << " seed: " << self.parameters.seed << std::endl; return out; } diff --git a/src/nupic/engine/Spec.cpp b/src/nupic/engine/Spec.cpp index bd6218ed35..237757863e 100644 --- a/src/nupic/engine/Spec.cpp +++ b/src/nupic/engine/Spec.cpp @@ -220,7 +220,7 @@ std::string Spec::toString() const { } std::ostream& operator<< (std::ostream& out, const Spec& self) { - out << self.toString(); + out << self.toString() << std::endl; return out; } diff --git a/src/nupic/types/Sdr.hpp b/src/nupic/types/Sdr.hpp index c5e2aed99c..9a2e2d6948 100644 --- a/src/nupic/types/Sdr.hpp +++ b/src/nupic/types/Sdr.hpp @@ -584,47 +584,6 @@ class SparseDistributedRepresentation : public Serializable } return stream << std::endl; } - friend std::istream& operator>> (std::istream& inStream, SparseDistributedRepresentation &sdr) - { - auto readVector = [&inStream] (std::vector &vec) { - inStream >> std::skipws; // skip leading whitespace - vec.clear(); - std::string val; - int c = 0; - while(c != '\n' && !inStream.eofbit) { - int c = inStream.get(); - if (isdigit(c)) val.append(1, c); - else if (c == ',') { - vec.push_back( atoi(val.c_str()) ); - val.clear(); - } - else if (c != ' ') break; - } - if (!val.empty()) - vec.push_back( atoi(val.c_str()) ); - }; - - // Read the starting marker and version. - std::string tag; - inStream >> tag; - NTA_CHECK( tag == "SDR(" ) << "stream does not contain an SDR from <<."; - - // Read the dimensions. - readVector( sdr.dimensions_ ); - - // Initialize the SDR. - // Calculate the SDR's size. - sdr.size_ = 1; - for(UInt dim : sdr.dimensions_) - sdr.size_ *= dim; - // Initialize sparse tuple. - sdr.coordinates_.assign( sdr.dimensions_.size(), {} ); - - // Read the data. - readVector( sdr.sparse_ ); - sdr.setSparseInplace(); - return inStream; - } bool operator==(const SparseDistributedRepresentation &sdr) const; inline bool operator!=(const SparseDistributedRepresentation &sdr) const diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 15f28e18bf..cbc2b748b8 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -1866,8 +1866,6 @@ TEST(SpatialPoolerTest, testSaveLoad) { ofstream outfile; outfile.open(filename, ifstream::binary); - outfile.precision(std::numeric_limits::digits10 + 1); - outfile.precision(std::numeric_limits::digits10 + 1); sp1.save(outfile); outfile.close(); @@ -2088,24 +2086,22 @@ TEST(SpatialPoolerTest, testConstructorVsInitialize) { } TEST(SpatialPoolerTest, ExactOutput) { - /**** TODO: fix it. - SDR copper_sdr(); - string copper = "SDR( 200 ) 190, 172, 23, 118, 178, 129, 113, 71, 185, 182\n"; - std::stringstream copper_stream( copper ); - copper_stream >> copper_sdr; - ****/ - + // Silver is an SDR that is loaded by direct initalization from a vector. SDR silver_sdr({ 200 }); SDR_sparse_t data = {190, 172, 23, 118, 178, 129, 113, 71, 185, 182}; silver_sdr.setSparse(data); - //silver_sdr.save(std::cout, JSON); + + // Gold tests initalizing an SDR from a manually created string in JSON format. + // hint: you can generate this string using + // silver_sdr.save(std::cout, JSON); string gold = "{\"dimensions\": [200],\"sparse\": [190,172,23,118,178,129,113,71,185,182]}"; std::stringstream gold_stream( gold ); SDR gold_sdr; gold_sdr.load( gold_stream, JSON ); + + // both SCR's should be the same EXPECT_EQ(silver_sdr, gold_sdr); - //EXPECT_EQ(copper_sdr, gold_sdr); SDR inputs({ 1000 }); SDR columns({ 200 }); diff --git a/src/test/unit/engine/YAMLUtilsTest.cpp b/src/test/unit/engine/YAMLUtilsTest.cpp index 4f8f620fae..2db516b9a0 100644 --- a/src/test/unit/engine/YAMLUtilsTest.cpp +++ b/src/test/unit/engine/YAMLUtilsTest.cpp @@ -182,9 +182,7 @@ TEST(YAMLUtilsTest, ParameterSpec) { << __FILE__ << ":" << __LINE__; ASSERT_FALSE(vm.getScalarT("boolParam")); - // disabled until we fix default string params - // TEST(vm.contains("stringParam")); - // EXPECT_STREQ("default value", vm.getString("stringParam")->c_str()); + EXPECT_STREQ("default value", vm.getString("stringParam").c_str()); // Test error message in case of invalid parameter with and without nodeType // and regionName From f8cfb95a0c40136fb7074fbf3c7ff3239ee217fb Mon Sep 17 00:00:00 2001 From: David Keeney Date: Sat, 8 Jun 2019 07:32:29 -0700 Subject: [PATCH 8/9] redo of changes to ScalarEncoder.cpp --- src/nupic/encoders/ScalarEncoder.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/nupic/encoders/ScalarEncoder.cpp b/src/nupic/encoders/ScalarEncoder.cpp index 0611494de8..ded8945234 100644 --- a/src/nupic/encoders/ScalarEncoder.cpp +++ b/src/nupic/encoders/ScalarEncoder.cpp @@ -197,15 +197,14 @@ void ScalarEncoder::encode(Real64 input, SDR &output) std::ostream & operator<<(std::ostream & out, const ScalarEncoder &self) { - out << "ScalarEncoder "; - out << "minimum: " << self.parameters.minimum << ", "; - out << "maximum: " << self.parameters.maximum << ", "; - out << "clipInput: " << self.parameters.clipInput << ", "; - out << "periodic: " << self.parameters.periodic << ", "; - out << "category: " << self.parameters.category << ", "; - out << "activeBits: " << self.parameters.activeBits << ", "; - // Save the resolution instead of the size BC it's higher precision. - out << "resolution: " << self.parameters.resolution << std::endl; + out << "ScalarEncoder \n"; + out << " minimum: " << self.parameters.minimum << ",\n"; + out << " maximum: " << self.parameters.maximum << ",\n"; + out << " clipInput: " << self.parameters.clipInput << ",\n"; + out << " periodic: " << self.parameters.periodic << ",\n"; + out << " category: " << self.parameters.category << ",\n"; + out << " activeBits: " << self.parameters.activeBits << ",\n"; + out << " resolution: " << self.parameters.resolution << std::endl; return out; } From 3d204cccb3bd77af557c8a14f07d40bdfd30b05c Mon Sep 17 00:00:00 2001 From: David Keeney Date: Sat, 8 Jun 2019 07:35:31 -0700 Subject: [PATCH 9/9] oops, need to column-ize it. --- src/nupic/encoders/ScalarEncoder.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nupic/encoders/ScalarEncoder.cpp b/src/nupic/encoders/ScalarEncoder.cpp index ded8945234..efbdf2af9c 100644 --- a/src/nupic/encoders/ScalarEncoder.cpp +++ b/src/nupic/encoders/ScalarEncoder.cpp @@ -198,13 +198,13 @@ void ScalarEncoder::encode(Real64 input, SDR &output) std::ostream & operator<<(std::ostream & out, const ScalarEncoder &self) { out << "ScalarEncoder \n"; - out << " minimum: " << self.parameters.minimum << ",\n"; - out << " maximum: " << self.parameters.maximum << ",\n"; + out << " minimum: " << self.parameters.minimum << ",\n"; + out << " maximum: " << self.parameters.maximum << ",\n"; out << " clipInput: " << self.parameters.clipInput << ",\n"; - out << " periodic: " << self.parameters.periodic << ",\n"; - out << " category: " << self.parameters.category << ",\n"; - out << " activeBits: " << self.parameters.activeBits << ",\n"; - out << " resolution: " << self.parameters.resolution << std::endl; + out << " periodic: " << self.parameters.periodic << ",\n"; + out << " category: " << self.parameters.category << ",\n"; + out << " activeBits:" << self.parameters.activeBits << ",\n"; + out << " resolution:" << self.parameters.resolution << std::endl; return out; }