Bp multi aggregation pippenger

contrib/fuzz_testing/fuzz.sh

@@ -10,12 +10,12 @@ fi
 type="$1"
 if test -z "$type"
 then
-  echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin"
+  echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"
   exit 1
 fi
 case "$type" in
-  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin) ;;
-  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin"; exit 1 ;;
+  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;;
+  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;;
 esac
 
 if test -d "fuzz-out/$type"

src/blockchain_db/berkeleydb/db_bdb.cpp

@@ -224,7 +224,7 @@ struct Dbt_safe : public Dbt
 namespace cryptonote
 {
 
-void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash)
+void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash)
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
     check_open();
@@ -255,7 +255,7 @@ void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const
     if (res)
         throw0(DB_ERROR("Failed to add block blob to db transaction."));
 
-    Dbt_copy<size_t> sz(block_size);
+    Dbt_copy<size_t> sz(block_weight);
     if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0))
         throw0(DB_ERROR("Failed to add block size to db transaction."));
 
@@ -1353,7 +1353,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const
     return get_block_timestamp(m_height - 1);
 }
 
-size_t BlockchainBDB::get_block_size(const uint64_t& height) const
+size_t BlockchainBDB::get_block_weight(const uint64_t& height) const
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
     check_open();
@@ -1861,7 +1861,7 @@ void BlockchainBDB::block_txn_abort()
   // TODO
 }
 
-uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs)
+uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs)
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
     check_open();
@@ -1874,7 +1874,7 @@ uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, co
     uint64_t num_outputs = m_num_outputs;
     try
     {
-        BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
+        BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs);
         m_write_txn = NULL;
 
         TIME_MEASURE_START(time1);

src/blockchain_db/berkeleydb/db_bdb.h

@@ -266,7 +266,7 @@ class BlockchainBDB : public BlockchainDB
 
   virtual uint64_t get_top_block_timestamp() const;
 
-  virtual size_t get_block_size(const uint64_t& height) const;
+  virtual size_t get_block_weight(const uint64_t& height) const;
 
   virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
 
@@ -318,7 +318,7 @@ class BlockchainBDB : public BlockchainDB
   virtual bool has_key_image(const crypto::key_image& img) const;
 
   virtual uint64_t add_block( const block& blk
-                            , const size_t& block_size
+                            , size_t block_weight
                             , const difficulty_type& cumulative_difficulty
                             , const uint64_t& coins_generated
                             , const std::vector<transaction>& txs
@@ -353,7 +353,7 @@ class BlockchainBDB : public BlockchainDB
 
 private:
   virtual void add_block( const block& blk
-                , const size_t& block_size
+                , size_t block_weight
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
                 , const crypto::hash& block_hash

src/blockchain_db/blockchain_db.cpp

@@ -196,7 +196,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
 }
 
 uint64_t BlockchainDB::add_block( const block& blk
-                                , const size_t& block_size
+                                , size_t block_weight
                                 , const difficulty_type& cumulative_difficulty
                                 , const uint64_t& coins_generated
                                 , const std::vector<transaction>& txs
@@ -241,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk
 
   // call out to subclass implementation to add the block & metadata
   time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
+  add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
   TIME_MEASURE_FINISH(time1);
   time_add_block1 += time1;
 

src/blockchain_db/blockchain_db.h

@@ -137,7 +137,7 @@ struct txpool_tx_meta_t
 {
   crypto::hash max_used_block_id;
   crypto::hash last_failed_id;
-  uint64_t blob_size;
+  uint64_t weight;
   uint64_t fee;
   uint64_t max_used_block_height;
   uint64_t last_failed_height;
@@ -358,13 +358,13 @@ class BlockchainDB
    * subclass of DB_EXCEPTION
    *
    * @param blk the block to be added
-   * @param block_size the size of the block (transactions and all)
+   * @param block_weight the weight of the block (transactions and all)
    * @param cumulative_difficulty the accumulated difficulty after this block
    * @param coins_generated the number of coins generated total after this block
    * @param blk_hash the hash of the block
    */
   virtual void add_block( const block& blk
-                , const size_t& block_size
+                , size_t block_weight
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs
@@ -376,7 +376,7 @@ class BlockchainDB
    *
    * The subclass implementing this will remove the block data from the top
    * block in the chain.  The data to be removed is that which was added in
-   * BlockchainDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash)
+   * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash)
    *
    * If any of this cannot be done, the subclass should throw the corresponding
    * subclass of DB_EXCEPTION
@@ -789,15 +789,15 @@ class BlockchainDB
    * subclass of DB_EXCEPTION
    *
    * @param blk the block to be added
-   * @param block_size the size of the block (transactions and all)
+   * @param block_weight the size of the block (transactions and all)
    * @param cumulative_difficulty the accumulated difficulty after this block
    * @param coins_generated the number of coins generated total after this block
    * @param txs the transactions in the block
    *
    * @return the height of the chain post-addition
    */
   virtual uint64_t add_block( const block& blk
-                            , const size_t& block_size
+                            , size_t block_weight
                             , const difficulty_type& cumulative_difficulty
                             , const uint64_t& coins_generated
                             , const std::vector<transaction>& txs
@@ -930,18 +930,18 @@ class BlockchainDB
   virtual uint64_t get_top_block_timestamp() const = 0;
 
   /**
-   * @brief fetch a block's size
+   * @brief fetch a block's weight
    *
-   * The subclass should return the size of the block with the
+   * The subclass should return the weight of the block with the
    * given height.
    *
    * If the block does not exist, the subclass should throw BLOCK_DNE
    *
    * @param height the height requested
    *
-   * @return the size
+   * @return the weight
    */
-  virtual size_t get_block_size(const uint64_t& height) const = 0;
+  virtual size_t get_block_weight(const uint64_t& height) const = 0;
 
   /**
    * @brief fetch a block's cumulative difficulty

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -256,7 +256,7 @@ typedef struct mdb_block_info_old
   uint64_t bi_height;
   uint64_t bi_timestamp;
   uint64_t bi_coins;
-  uint64_t bi_size; // a size_t really but we need 32-bit compat
+  uint64_t bi_weight; // a size_t really but we need 32-bit compat
   difficulty_type bi_diff;
   crypto::hash bi_hash;
 } mdb_block_info_old;
@@ -266,7 +266,7 @@ typedef struct mdb_block_info
   uint64_t bi_height;
   uint64_t bi_timestamp;
   uint64_t bi_coins;
-  uint64_t bi_size; // a size_t really but we need 32-bit compat
+  uint64_t bi_weight; // a size_t really but we need 32-bit compat
   difficulty_type bi_diff;
   crypto::hash bi_hash;
   uint64_t bi_cum_rct;
@@ -652,8 +652,11 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin
     block_rtxn_start(&rtxn, &rcurs);
     for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
     {
-      uint32_t block_size = get_block_size(block_num);
-      total_block_size += block_size;
+      // we have access to block weight, which will be greater or equal to block size,
+      // so use this as a proxy. If it's too much off, we might have to check actual size,
+      // which involves reading more data, so is not really wanted
+      size_t block_weight = get_block_weight(block_num);
+      total_block_size += block_weight;
       // Track number of blocks being totalled here instead of assuming, in case
       // some blocks were to be skipped for being outliers.
       ++num_blocks_used;
@@ -674,7 +677,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin
   return threshold_size;
 }
 
-void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
+void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
     uint64_t num_rct_outs, const crypto::hash& blk_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -720,7 +723,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   bi.bi_height = m_height;
   bi.bi_timestamp = blk.timestamp;
   bi.bi_coins = coins_generated;
-  bi.bi_size = block_size;
+  bi.bi_weight = block_weight;
   bi.bi_diff = cumulative_difficulty;
   bi.bi_hash = blk_hash;
   bi.bi_cum_rct = num_rct_outs;
@@ -743,7 +746,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   if (result)
     throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str()));
 
-  m_cum_size += block_size;
+  // we use weight as a proxy for size, since we don't have size but weight is >= size
+  // and often actually equal
+  m_cum_size += block_weight;
   m_cum_count++;
 }
 
@@ -2011,7 +2016,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const
   return get_block_timestamp(m_height - 1);
 }
 
-size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
+size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2029,7 +2034,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
     throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
 
   mdb_block_info *bi = (mdb_block_info *)result.mv_data;
-  size_t ret = bi->bi_size;
+  size_t ret = bi->bi_weight;
   TXN_POSTFIX_RDONLY();
   return ret;
 }
@@ -3090,7 +3095,7 @@ void BlockchainLMDB::block_txn_abort()
   }
 }
 
-uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
+uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
     const std::vector<transaction>& txs)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -3109,7 +3114,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
 
   try
   {
-    BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
+    BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs);
   }
   catch (const DB_ERROR_TXN_START& e)
   {
@@ -3698,9 +3703,9 @@ void BlockchainLMDB::migrate_0_1()
       if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str()));
       if (v.mv_size == sizeof(uint32_t))
-        bi.bi_size = *(uint32_t *)v.mv_data;
+        bi.bi_weight = *(uint32_t *)v.mv_data;
       else
-        bi.bi_size = *(uint64_t *)v.mv_data;  // this is a 32/64 compat bug in version 0
+        bi.bi_weight = *(uint64_t *)v.mv_data;  // this is a 32/64 compat bug in version 0
       result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT);
       if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str()));
@@ -4259,7 +4264,7 @@ void BlockchainLMDB::migrate_2_3()
       bi.bi_height = bi_old->bi_height;
       bi.bi_timestamp = bi_old->bi_timestamp;
       bi.bi_coins = bi_old->bi_coins;
-      bi.bi_size = bi_old->bi_size;
+      bi.bi_weight = bi_old->bi_weight;
       bi.bi_diff = bi_old->bi_diff;
       bi.bi_hash = bi_old->bi_hash;
       if (bi_old->bi_height >= distribution.size())

src/blockchain_db/lmdb/db_lmdb.h

@@ -205,7 +205,7 @@ class BlockchainLMDB : public BlockchainDB
 
   virtual uint64_t get_top_block_timestamp() const;
 
-  virtual size_t get_block_size(const uint64_t& height) const;
+  virtual size_t get_block_weight(const uint64_t& height) const;
 
   virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
 
@@ -273,7 +273,7 @@ class BlockchainLMDB : public BlockchainDB
   virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const;
 
   virtual uint64_t add_block( const block& blk
-                            , const size_t& block_size
+                            , size_t block_weight
                             , const difficulty_type& cumulative_difficulty
                             , const uint64_t& coins_generated
                             , const std::vector<transaction>& txs
@@ -317,7 +317,7 @@ class BlockchainLMDB : public BlockchainDB
   uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const;
 
   virtual void add_block( const block& blk
-                , const size_t& block_size
+                , size_t block_weight
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs

src/blockchain_utilities/blockchain_import.cpp

@@ -474,17 +474,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
             txs.push_back(tx);
           }
 
-          size_t block_size;
+          size_t block_weight;
           difficulty_type cumulative_difficulty;
           uint64_t coins_generated;
 
-          block_size = bp.block_size;
+          block_weight = bp.block_weight;
           cumulative_difficulty = bp.cumulative_difficulty;
           coins_generated = bp.coins_generated;
 
           try
           {
-            core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs);
+            core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs);
           }
           catch (const std::exception& e)
           {

src/blockchain_utilities/bootstrap_file.cpp

@@ -236,11 +236,11 @@ void BootstrapFile::write_block(block& block)
   bool include_extra_block_data = true;
   if (include_extra_block_data)
   {
-    size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height);
+    size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height);
     difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
     uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
 
-    bp.block_size = block_size;
+    bp.block_weight = block_weight;
     bp.cumulative_difficulty = cumulative_difficulty;
     bp.coins_generated = coins_generated;
   }

src/blockchain_utilities/bootstrap_serialization.h

@@ -70,14 +70,14 @@ namespace cryptonote
     {
       cryptonote::block block;
       std::vector<transaction> txs;
-      size_t block_size;
+      size_t block_weight;
       difficulty_type cumulative_difficulty;
       uint64_t coins_generated;
 
       BEGIN_SERIALIZE()
         FIELD(block)
         FIELD(txs)
-        VARINT_FIELD(block_size)
+        VARINT_FIELD(block_weight)
         VARINT_FIELD(cumulative_difficulty)
         VARINT_FIELD(coins_generated)
       END_SERIALIZE()

src/common/CMakeLists.txt

@@ -38,7 +38,8 @@ set(common_sources
   password.cpp
   perf_timer.cpp
   threadpool.cpp
-  updates.cpp)
+  updates.cpp
+  aligned.c)
 
 if (STACK_TRACE)
   list(APPEND common_sources stack_trace.cpp)
@@ -67,7 +68,8 @@ set(common_private_headers
   perf_timer.h
   stack_trace.h
   threadpool.h
-  updates.h)
+  updates.h
+  aligned.h)
 
 monero_private_headers(common
   ${common_private_headers})