rbrunner7 · rbrunner7 · Jun 28, 2024 · Nov 2, 2018 · Nov 23, 2018 · Nov 24, 2018
diff --git a/cmake/CheckTrezor.cmake b/cmake/CheckTrezor.cmake
@@ -56,7 +56,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
         endif()
 
         if(USE_DEVICE_TREZOR_UDP_RELEASE)
-            add_definitions(-DWITH_DEVICE_TREZOR_UDP_RELEASE=1)
+            add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1)
         endif()
 
         if (Protobuf_INCLUDE_DIR)

diff --git a/contrib/depends/packages/bdb.mk b/contrib/depends/packages/bdb.mk
@@ -1,6 +1,6 @@
 package=bdb
 $(package)_version=4.8.30
-$(package)_download_path=http://download.oracle.com/berkeley-db
+$(package)_download_path=https://download.oracle.com/berkeley-db
 $(package)_file_name=db-$($(package)_version).NC.tar.gz
 $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
 $(package)_build_subdir=build_unix

diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk
@@ -1,6 +1,6 @@
 package=ldns
 $(package)_version=1.6.17
-$(package)_download_path=http://www.nlnetlabs.nl/downloads/ldns/
+$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd
 $(package)_dependencies=openssl

diff --git a/contrib/depends/packages/libICE.mk b/contrib/depends/packages/libICE.mk
@@ -1,6 +1,6 @@
 package=libICE
 $(package)_version=1.0.9
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=8f7032f2c1c64352b5423f6b48a8ebdc339cc63064af34d66a6c9aa79759e202
 $(package)_dependencies=xtrans xproto

diff --git a/contrib/depends/packages/libSM.mk b/contrib/depends/packages/libSM.mk
@@ -1,6 +1,6 @@
 package=libSM
 $(package)_version=1.2.2
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=0baca8c9f5d934450a70896c4ad38d06475521255ca63b717a6510fdb6e287bd
 $(package)_dependencies=xtrans xproto libICE

diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk
@@ -1,6 +1,6 @@
 package=libusb
 $(package)_version=1.0.22
-$(package)_download_path=http://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
+$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
 

diff --git a/contrib/depends/packages/native_cdrkit.mk b/contrib/depends/packages/native_cdrkit.mk
@@ -1,6 +1,6 @@
 package=native_cdrkit
 $(package)_version=1.1.11
-$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c
+$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c
 $(package)_file_name=cdrkit-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
 $(package)_patches=cdrkit-deterministic.patch

diff --git a/contrib/depends/packages/qt.mk b/contrib/depends/packages/qt.mk
@@ -1,6 +1,6 @@
 PACKAGE=qt
 $(package)_version=5.7.1
-$(package)_download_path=http://download.qt.io/official_releases/qt/5.7/$($(package)_version)/submodules
+$(package)_download_path=https://download.qt.io/archive/qt/5.7/5.7.1/submodules
 $(package)_suffix=opensource-src-$($(package)_version).tar.gz
 $(package)_file_name=qtbase-$($(package)_suffix)
 $(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410

diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk
@@ -1,6 +1,6 @@
 package=unbound
 $(package)_version=1.6.8
-$(package)_download_path=http://www.unbound.net/downloads/
+$(package)_download_path=https://www.unbound.net/downloads/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49
 $(package)_dependencies=openssl expat ldns

diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk
@@ -1,6 +1,6 @@
 package=unwind
 $(package)_version=1.2
-$(package)_download_path=http://download.savannah.nongnu.org/releases/libunwind
+$(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind
 $(package)_file_name=lib$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992
 

diff --git a/contrib/depends/packages/xproto.mk b/contrib/depends/packages/xproto.mk
@@ -1,6 +1,6 @@
 package=xproto
 $(package)_version=7.0.26
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/proto
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f
 

diff --git a/contrib/depends/packages/zlib.mk b/contrib/depends/packages/zlib.mk
@@ -1,6 +1,6 @@
 package=zlib
 $(package)_version=1.2.11
-$(package)_download_path=http://www.zlib.net
+$(package)_download_path=https://www.zlib.net
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
 

diff --git a/contrib/epee/src/mlocker.cpp b/contrib/epee/src/mlocker.cpp
@@ -53,7 +53,6 @@ static size_t query_page_size()
     MERROR("Failed to determine page size");
     return 0;
   }
-  MINFO("Page size: " << ret);
   return ret;
 #else
 #warning Missing query_page_size implementation

diff --git a/contrib/valgrind/monero.supp b/contrib/valgrind/monero.supp
@@ -17,3 +17,12 @@
    fun:maybe_unlock_and_signal_one<boost::asio::detail::scoped_lock<boost::asio::detail::posix_mutex> >
    ...
 }
+
+{
+   we leak the logger, for performance reasons in on-the-fly init
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:_Znwm
+   fun:_ZN2el4base7Storage7getELPPEv
+   ...
+}
diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc
@@ -2191,17 +2191,13 @@ void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs)
 #   define ELPP_DEFAULT_LOGGING_FLAGS 0x0
 #endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS)
 // Storage
-el::base::type::StoragePointer getresetELPP(bool reset)
+el::base::type::StoragePointer &el::base::Storage::getELPP()
 {
-  static el::base::type::StoragePointer p(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder())));
-  if (reset)
-    p = NULL;
-  return p;
-}
-el::base::type::StoragePointer el::base::Storage::getELPP()
-{
-  return getresetELPP(false);
+  if (!el::base::elStorage)
+    el::base::elStorage = new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()));
+  return el::base::elStorage;
 }
+static struct EnsureELPP { EnsureELPP() { el::base::Storage::getELPP(); } } ensureELPP;
 #if ELPP_ASYNC_LOGGING
 Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) :
 #else
@@ -2250,7 +2246,6 @@ Storage::Storage(const LogBuilderPtr& defaultLogBuilder) :
 
 Storage::~Storage(void) {
   ELPP_INTERNAL_INFO(4, "Destroying storage");
-  getresetELPP(true);
 #if ELPP_ASYNC_LOGGING
   ELPP_INTERNAL_INFO(5, "Replacing log dispatch callback to synchronous");
   uninstallLogDispatchCallback<base::AsyncLogDispatchCallback>(std::string("AsyncLogDispatchCallback"));

diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h
@@ -552,7 +552,7 @@ typedef std::ostream ostream_t;
 typedef unsigned int EnumType;
 typedef unsigned short VerboseLevel;
 typedef unsigned long int LineNumber;
-typedef std::shared_ptr<base::Storage> StoragePointer;
+typedef base::Storage *StoragePointer;
 typedef std::shared_ptr<LogDispatchCallback> LogDispatchCallbackPtr;
 typedef std::shared_ptr<PerformanceTrackingCallback> PerformanceTrackingCallbackPtr;
 typedef std::shared_ptr<LoggerRegistrationCallback> LoggerRegistrationCallbackPtr;
@@ -2734,7 +2734,7 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe {
     return it->second;
   }
 
-  static el::base::type::StoragePointer getELPP();
+  static el::base::type::StoragePointer &getELPP();
 
  private:
   base::RegisteredHitCounters* m_registeredHitCounters;
@@ -4613,9 +4613,10 @@ el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \
 }
 
 #if ELPP_ASYNC_LOGGING
-#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(NULL)
+#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()),\
+new el::base::AsyncDispatchWorker()))
 #else
-#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(NULL)
+#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder())))
 #endif  // ELPP_ASYNC_LOGGING
 #define INITIALIZE_NULL_EASYLOGGINGPP \
 namespace el {\

diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
@@ -1404,6 +1404,13 @@ class BlockchainDB
    */
   virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0;
 
+  /**
+   * @brief prune output data for the given amount
+   *
+   * @param amount the amount for which to prune data
+   */
+  virtual void prune_outputs(uint64_t amount) = 0;
+
   /**
    * @brief runs a function over all txpool transactions
    *

diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -1075,6 +1075,60 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in
     throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
 }
 
+void BlockchainLMDB::prune_outputs(uint64_t amount)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+  CURSOR(output_amounts);
+  CURSOR(output_txs);
+
+  MINFO("Pruning outputs for amount " << amount);
+
+  MDB_val v;
+  MDB_val_set(k, amount);
+  int result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
+  if (result == MDB_NOTFOUND)
+    return;
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Error looking up outputs: ", result).c_str()));
+
+  // gather output ids
+  mdb_size_t num_elems;
+  mdb_cursor_count(m_cur_output_amounts, &num_elems);
+  MINFO(num_elems << " outputs found");
+  std::vector<uint64_t> output_ids;
+  output_ids.reserve(num_elems);
+  while (1)
+  {
+    const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
+    output_ids.push_back(okp->output_id);
+    MDEBUG("output id " << okp->output_id);
+    result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP);
+    if (result == MDB_NOTFOUND)
+      break;
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error counting outputs: ", result).c_str()));
+  }
+  if (output_ids.size() != num_elems)
+    throw0(DB_ERROR("Unexpected number of outputs"));
+
+  result = mdb_cursor_del(m_cur_output_amounts, MDB_NODUPDATA);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Error deleting outputs: ", result).c_str()));
+
+  for (uint64_t output_id: output_ids)
+  {
+    MDB_val_set(v, output_id);
+    result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error looking up output: ", result).c_str()));
+    result = mdb_cursor_del(m_cur_output_txs, 0);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error deleting output: ", result).c_str()));
+  }
+}
+
 void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -2229,11 +2283,19 @@ uint64_t BlockchainLMDB::num_outputs() const
   TXN_PREFIX_RDONLY();
   int result;
 
-  // get current height
-  MDB_stat db_stats;
-  if ((result = mdb_stat(m_txn, m_output_txs, &db_stats)))
+  RCURSOR(output_txs)
+
+  uint64_t num = 0;
+  MDB_val k, v;
+  result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_LAST);
+  if (result == MDB_NOTFOUND)
+    num = 0;
+  else if (result == 0)
+    num = 1 + ((const outtx*)v.mv_data)->output_id;
+  else
     throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
-  return db_stats.ms_entries;
+
+  return num;
 }
 
 bool BlockchainLMDB::tx_exists(const crypto::hash& h) const

diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
@@ -345,6 +345,8 @@ class BlockchainLMDB : public BlockchainDB
 
   void remove_output(const uint64_t amount, const uint64_t& out_index);
 
+  virtual void prune_outputs(uint64_t amount);
+
   virtual void add_spent_key(const crypto::key_image& k_image);
 
   virtual void remove_spent_key(const crypto::key_image& k_image);

diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt
@@ -81,6 +81,17 @@ monero_private_headers(blockchain_usage
 
 
 
+set(blockchain_prune_known_spent_data_sources
+  blockchain_prune_known_spent_data.cpp
+  )
+
+set(blockchain_prune_known_spent_data_private_headers)
+
+monero_private_headers(blockchain_prune_known_spent_data
+	  ${blockchain_prune_known_spent_data_private_headers})
+
+
+
 set(blockchain_ancestry_sources
   blockchain_ancestry.cpp
   )
@@ -265,3 +276,25 @@ set_property(TARGET blockchain_stats
 	PROPERTY
 	OUTPUT_NAME "monero-blockchain-stats")
 install(TARGETS blockchain_stats DESTINATION bin)
+
+monero_add_executable(blockchain_prune_known_spent_data
+  ${blockchain_prune_known_spent_data_sources}
+  ${blockchain_prune_known_spent_data_private_headers})
+
+target_link_libraries(blockchain_prune_known_spent_data
+  PRIVATE
+    cryptonote_core
+    blockchain_db
+    p2p
+    version
+    epee
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES})
+
+set_property(TARGET blockchain_prune_known_spent_data
+	PROPERTY
+	OUTPUT_NAME "monero-blockchain-prune-known-spent-data")
+install(TARGETS blockchain_prune_known_spent_data DESTINATION bin)