1347 data destruction

libbatched-io/batched_db.cpp

@@ -59,4 +59,15 @@ void db_splitter::prefixed_db::forEach(
     } );
 }
 
+void db_splitter::prefixed_db::forEachWithPrefix(
+    std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const {
+    backend->forEachWithPrefix( _prefix, [&]( dev::db::Slice _key, dev::db::Slice _val ) -> bool {
+        if ( _key[0] != this->prefix )
+            return true;
+        dev::db::Slice key_short = dev::db::Slice( _key.data() + 1, _key.size() - 1 );
+        return f( key_short, _val );
+    } );
+}
+
+
 }  // namespace batched_io

libbatched-io/batched_db.h

@@ -18,7 +18,8 @@ class db_operations_face {
     virtual std::string lookup( dev::db::Slice _key ) const = 0;
     virtual bool exists( dev::db::Slice _key ) const = 0;
     virtual void forEach( std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const = 0;
-
+    virtual void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const = 0;
     virtual ~db_operations_face() = default;
 };
 
@@ -69,6 +70,12 @@ class batched_db : public db_face {
         m_db->forEach( f );
     }
 
+    virtual void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const {
+        std::lock_guard< std::mutex > foreach_lock( m_batch_mutex );
+        m_db->forEachWithPrefix( _prefix, f );
+    }
+
     virtual ~batched_db();
 
 protected:
@@ -105,6 +112,8 @@ class db_splitter {
         virtual std::string lookup( dev::db::Slice _key ) const;
         virtual bool exists( dev::db::Slice _key ) const;
         virtual void forEach( std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const;
+        virtual void forEachWithPrefix(
+            std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const;
 
     protected:
         virtual void recover() { /* nothing */

libdevcore/LevelDB.cpp

@@ -198,6 +198,26 @@ void LevelDB::forEach( std::function< bool( Slice, Slice ) > f ) const {
     }
 }
 
+
+void LevelDB::forEachWithPrefix(
+    std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
+    cwarn << "Iterating over the entire LevelDB database: " << this->m_path;
+    std::unique_ptr< leveldb::Iterator > itr( m_db->NewIterator( m_readOptions ) );
+    if ( itr == nullptr ) {
+        BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
+    }
+    auto keepIterating = true;
+    auto prefixSlice = leveldb::Slice( _prefix );
+    for ( itr->Seek( prefixSlice );
+          keepIterating && itr->Valid() && itr->key().starts_with( prefixSlice ); itr->Next() ) {
+        auto const dbKey = itr->key();
+        auto const dbValue = itr->value();
+        Slice const key( dbKey.data(), dbKey.size() );
+        Slice const value( dbValue.data(), dbValue.size() );
+        keepIterating = f( key, value );
+    }
+}
+
 h256 LevelDB::hashBase() const {
     std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
     if ( it == nullptr ) {

libdevcore/LevelDB.h

@@ -52,6 +52,9 @@ class LevelDB : public DatabaseFace {
 
     void forEach( std::function< bool( Slice, Slice ) > f ) const override;
 
+    void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const override;
+
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 

libdevcore/ManuallyRotatingLevelDB.cpp

@@ -67,6 +67,15 @@ void ManuallyRotatingLevelDB::forEach( std::function< bool( Slice, Slice ) > f )
     }
 }
 
+void ManuallyRotatingLevelDB::forEachWithPrefix(
+    std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
+    std::shared_lock< std::shared_mutex > lock( m_mutex );
+    for ( const auto& p : *io_backend ) {
+        p->forEachWithPrefix( _prefix, f );
+    }
+}
+
+
 h256 ManuallyRotatingLevelDB::hashBase() const {
     std::shared_lock< std::shared_mutex > lock( m_mutex );
     secp256k1_sha256_t ctx;

libdevcore/ManuallyRotatingLevelDB.h

@@ -41,6 +41,9 @@ class ManuallyRotatingLevelDB : public DatabaseFace {
     }
 
     virtual void forEach( std::function< bool( Slice, Slice ) > f ) const;
+    virtual void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const;
+
     virtual h256 hashBase() const;
 };
 

libdevcore/SplitDB.cpp

@@ -100,6 +100,20 @@ void SplitDB::PrefixedDB::forEach( std::function< bool( Slice, Slice ) > f ) con
     } );
 }
 
+
+void SplitDB::PrefixedDB::forEachWithPrefix(
+    std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
+    std::unique_lock< std::shared_mutex > lock( this->backend_mutex );
+    auto prefixedString = std::to_string( this->prefix ) + _prefix;
+    backend->forEachWithPrefix( prefixedString, [&]( Slice _key, Slice _val ) -> bool {
+        if ( _key[0] != this->prefix )
+            return true;
+        Slice key_short = Slice( _key.data() + 1, _key.size() - 1 );
+        return f( key_short, _val );
+    } );
+}
+
+
 h256 SplitDB::PrefixedDB::hashBase() const {
     // HACK TODO implement that it would work with any DatabaseFace*
     const LevelDB* ldb = dynamic_cast< const LevelDB* >( backend.get() );

libdevcore/SplitDB.h

@@ -36,6 +36,9 @@ class SplitDB {
         virtual void commit( std::unique_ptr< WriteBatchFace > _batch );
 
         virtual void forEach( std::function< bool( Slice, Slice ) > f ) const;
+        virtual void forEachWithPrefix(
+            std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const;
+
         virtual h256 hashBase() const;
 
     private:

libdevcore/db.h

@@ -66,6 +66,11 @@ class DatabaseFace {
     // of each record in the database. If `f` returns false, the `forEach`
     // method must return immediately.
     virtual void forEach( std::function< bool( Slice, Slice ) > f ) const = 0;
+
+    virtual void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const = 0;
+
+
     virtual h256 hashBase() const = 0;
 
     virtual bool discardCreatedBatches() { return false; }

libethcore/ChainOperationParams.h

@@ -173,6 +173,7 @@ struct SChain {
     time_t contractStoragePatchTimestamp = 0;
     time_t contractStorageZeroValuePatchTimestamp = 0;
     time_t verifyDaSigsPatchTimestamp = 0;
+    time_t storageDestructionPatchTimestamp = 0;
 
     SChain() {
         name = "TestChain";

libethereum/ChainParams.cpp

@@ -257,6 +257,11 @@ ChainParams ChainParams::loadConfig(
                 sChainObj.at( "verifyDaSigsPatchTimestamp" ).get_int64() :
                 0;
 
+        s.storageDestructionPatchTimestamp =
+            sChainObj.count( "storageDestructionPatchTimestamp" ) ?
+                sChainObj.at( "storageDestructionPatchTimestamp" ).get_int64() :
+                0;
+
         if ( sChainObj.count( "nodeGroups" ) ) {
             std::vector< NodeGroup > nodeGroups;
             for ( const auto& nodeGroupConf : sChainObj["nodeGroups"].get_obj() ) {

libethereum/Client.cpp

@@ -52,6 +52,7 @@
 #include <libskale/ContractStorageZeroValuePatch.h>
 #include <libskale/RevertableFSPatch.h>
 #include <libskale/State.h>
+#include <libskale/StorageDestructionPatch.h>
 #include <libskale/TotalStorageUsedPatch.h>
 #include <libskale/UnsafeRegion.h>
 #include <libskale/VerifyDaSigsPatch.h>
@@ -155,6 +156,8 @@ Client::Client( ChainParams const& _params, int _networkID,
         chainParams().sChain.contractStorageZeroValuePatchTimestamp;
     VerifyDaSigsPatch::verifyDaSigsPatchTimestamp = chainParams().sChain.verifyDaSigsPatchTimestamp;
     RevertableFSPatch::revertableFSPatchTimestamp = chainParams().sChain.revertableFSPatchTimestamp;
+    StorageDestructionPatch::storageDestructionPatchTimestamp =
+        chainParams().sChain.storageDestructionPatchTimestamp;
 }
 
 Client::~Client() {
@@ -747,6 +750,7 @@ size_t Client::syncTransactions(
     ContractStorageLimitPatch::lastBlockTimestamp = blockChain().info().timestamp();
     ContractStorageZeroValuePatch::lastBlockTimestamp = blockChain().info().timestamp();
     RevertableFSPatch::lastBlockTimestamp = blockChain().info().timestamp();
+    StorageDestructionPatch::lastBlockTimestamp = blockChain().info().timestamp();
 
 
     DEV_WRITE_GUARDED( x_working ) {

libethereum/Executive.cpp

@@ -55,7 +55,8 @@ std::string dumpStackAndMemory( LegacyVM const& _vm ) {
 std::string dumpStorage( ExtVM const& _ext ) {
     ostringstream o;
     o << "    STORAGE\n";
-    for ( auto const& i : _ext.state().storage( _ext.myAddress ) )
+    // this function is called when a lock over the state already is acquired
+    for ( auto const& i : _ext.state().storage_WITHOUT_LOCK( _ext.myAddress ) )
         o << showbase << hex << i.second.first << ": " << i.second.second << "\n";
     return o.str();
 }

libskale/CMakeLists.txt

@@ -18,6 +18,7 @@ set(sources
     AmsterdamFixPatch.cpp
     RevertableFSPatch.cpp
     OverlayFS.cpp
+    StorageDestructionPatch.cpp
 )
 
 set(headers

libskale/OverlayDB.cpp

@@ -330,7 +330,9 @@ std::unordered_map< h160, string > OverlayDB::accounts() const {
 std::unordered_map< u256, u256 > OverlayDB::storage( const dev::h160& _address ) const {
     unordered_map< u256, u256 > storage;
     if ( m_db_face ) {
-        m_db_face->forEach( [&storage, &_address]( Slice key, Slice value ) {
+        // iterate of a keys that start with the given substring
+        string prefix( ( const char* ) _address.data(), _address.size );
+        m_db_face->forEachWithPrefix( prefix, [&storage, &_address]( Slice key, Slice value ) {
             if ( key.size() == h160::size + h256::size ) {
                 // key is storage address
                 string keyString( key.begin(), key.end() );
@@ -342,6 +344,8 @@ std::unordered_map< u256, u256 > OverlayDB::storage( const dev::h160& _address )
                     u256 memoryValue = h256( string( value.begin(), value.end() ),
                         h256::ConstructFromStringType::FromBinary );
                     storage[memoryAddress] = memoryValue;
+                } else {
+                    cerror << "Address mismatch in:" << __FUNCTION__;
                 }
             }
             return true;

libskale/State.cpp

@@ -46,6 +46,7 @@
 
 #include <libethereum/BlockDetails.h>
 #include <libskale/RevertableFSPatch.h>
+#include <libskale/StorageDestructionPatch.h>
 
 namespace fs = boost::filesystem;
 
@@ -508,7 +509,11 @@ void State::commit( dev::eth::CommitBehaviour _commitBehaviour ) {
                 if ( !account.isAlive() ) {
                     m_db_ptr->kill( address );
                     m_db_ptr->killAuxiliary( address, Auxiliary::CODE );
-                    // TODO: remove account storage
+
+                    if ( StorageDestructionPatch::isEnabled() ) {
+                        clearStorage( address );
+                    }
+
                 } else {
                     RLPStream rlpStream( 4 );
 
@@ -653,8 +658,14 @@ void State::kill( Address _addr ) {
     // If the account is not in the db, nothing to kill.
 }
 
+
 std::map< h256, std::pair< u256, u256 > > State::storage( const Address& _contract ) const {
     boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr );
+    return storage_WITHOUT_LOCK( _contract );
+}
+
+
+std::map< h256, std::pair< u256, u256 > > State::storage_WITHOUT_LOCK( const Address& _contract ) const {
     if ( !checkVersion() ) {
         cerror << "Current state version is " << m_currentVersion << " but stored version is "
                << *m_storedVersion << endl;
@@ -751,13 +762,6 @@ u256 State::originalStorageValue( Address const& _contract, u256 const& _key ) c
 }
 
 
-// Clear storage needs to be called when a new contract is
-// created for an address that included a different contract
-// that was destroyed using selfdestruct op code
-// The only way this can happen if one calls
-// CREATE2, self-destruct, and then CREATE2 again, which is
-// extremely rare and a bad security practice
-// Note that in Shanhai fork the selfdestruct op code will be removed
 void State::clearStorage( Address const& _contract ) {
     // only clear storage if the storage used is not 0
 
@@ -768,16 +772,17 @@ void State::clearStorage( Address const& _contract ) {
         return;
     }
 
-    // TODO: This is extremely inefficient
-    for ( auto const& hashPairPair : storage( _contract ) ) {
+    // clearStorage is called from functions that already hold a read
+    // or write lock over the state Therefore, we can use
+    // storage_WITHOUT_LOCK() here
+    for ( auto const& hashPairPair : storage_WITHOUT_LOCK( _contract ) ) {
         auto const& key = hashPairPair.second.first;
         setStorage( _contract, key, 0 );
         acc->setStorageCache( key, 0 );
     }
 
     totalStorageUsed_ -= ( accStorageUsed + storageUsage[_contract] );
     acc->updateStorageUsage( -accStorageUsed );
-    // TODO Do we need to clear storageUsage[_contract] here?
 }
 
 bytes const& State::code( Address const& _addr ) const {

libskale/State.h

@@ -292,6 +292,11 @@ class State {
     std::map< dev::h256, std::pair< dev::u256, dev::u256 > > storage(
         dev::Address const& _contract ) const;
 
+    /// Not thread safe version of storage() that does not hold any locks
+    std::map< dev::h256, std::pair< dev::u256, dev::u256 > > storage_WITHOUT_LOCK(
+        const Address& _contract ) const;
+
+
     /// Get the code of an account.
     /// @returns bytes() if no account exists at that address.
     /// @warning The reference to the code is only valid until the access to

libskale/StorageDestructionPatch.cpp

@@ -0,0 +1,8 @@
+#include "StorageDestructionPatch.h"
+
+time_t StorageDestructionPatch::storageDestructionPatchTimestamp;
+time_t StorageDestructionPatch::lastBlockTimestamp;
+
+bool StorageDestructionPatch::isEnabled() {
+    return storageDestructionPatchTimestamp <= lastBlockTimestamp;
+}

libskale/StorageDestructionPatch.h

@@ -0,0 +1,21 @@
+#include <libethereum/SchainPatch.h>
+#include <time.h>
+
+namespace dev {
+namespace eth {
+class Client;
+}
+}  // namespace dev
+
+/*
+ * Context: enable effective storage destruction
+ */
+class StorageDestructionPatch : public SchainPatch {
+public:
+    static bool isEnabled();
+
+private:
+    friend class dev::eth::Client;
+    static time_t storageDestructionPatchTimestamp;
+    static time_t lastBlockTimestamp;
+};