Ln/merge mem reduction into feature pos

.github/workflows/ci.yml

@@ -84,7 +84,8 @@ jobs:
  go test -run TestAssociations -v ./lib &&
  go test -run TestDAOCoinLimitOrder -v ./lib &&
  go test -run TestFreezingPosts -v ./lib &&
- go test -run TestBalanceModelAssociations -v ./lib
+ go test -run TestBalanceModelAssociations -v ./lib &&
+ go test -run TestPGGenesisBlock -v ./lib
 
  - name: Rollback migrations
  run: go run scripts/migrate.go rollback

integration_testing/tools.go

@@ -399,7 +399,8 @@ func listenForBlockHeight(t *testing.T, node *cmd.Node, height uint32, signal ch
  }()
 }
 
-// listenForBlockHeight busy-waits until the node's block tip reaches provided height.
+// disconnectAtBlockHeight busy-waits until the node's block tip reaches provided height, and then disconnects
+// from the provided bridge.
 func disconnectAtBlockHeight(t *testing.T, syncingNode *cmd.Node, bridge *ConnectionBridge, height uint32) {
  listener := make(chan bool)
  listenForBlockHeight(t, syncingNode, height, listener)

lib/block_view_bitcoin_test.go

@@ -74,7 +74,7 @@ func _dumpAndLoadMempool(t *testing.T, mempool *DeSoMempool) {
  newMempool := NewDeSoMempool(
  mempool.bc, 0, /* rateLimitFeeRateNanosPerKB */
  0 /* minFeeRateNanosPerKB */, "", true,
- mempool.dataDir, "")
+ mempool.dataDir, "", mempool.useDefaultBadgerOptions)
  mempool.mempoolDir = ""
  newMempool.mempoolDir = mempoolDir
  newMempool.LoadTxnsFromDB()
@@ -370,7 +370,7 @@ func TestBitcoinExchange(t *testing.T) {
  newMP := NewDeSoMempool(chain, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  mempool.resetPool(newMP)
 
  // Validating the first Bitcoin burn transaction via a UtxoView should
@@ -1076,7 +1076,7 @@ func TestBitcoinExchangeGlobalParams(t *testing.T) {
  newMP := NewDeSoMempool(chain, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  mempool.resetPool(newMP)
 
  //// Validating the first Bitcoin burn transaction via a UtxoView should
@@ -1808,7 +1808,7 @@ func TestSpendOffOfUnminedTxnsBitcoinExchange(t *testing.T) {
  newMP := NewDeSoMempool(chain, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  mempool.resetPool(newMP)
 
  // The amount of work on the first burn transaction should be zero.
@@ -2419,7 +2419,7 @@ func TestBitcoinExchangeWithAmountNanosNonZeroAtGenesis(t *testing.T) {
  newMP := NewDeSoMempool(chain, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  mempool.resetPool(newMP)
 
  // The amount of work on the first burn transaction should be zero.

lib/block_view_test.go

@@ -418,7 +418,7 @@ func (tes *transactionTestSuite) InitializeChainAndGetTestMeta(useBadger bool, u
  pgPort = config.testPostgresPort
  }
  chain, params, embpg = NewLowDifficultyBlockchainWithParamsAndDb(tes.t, &DeSoTestnetParams,
- true, pgPort)
+ true, pgPort, false)
  mempool, miner = NewTestMiner(config.t, chain, params, true /*isSender*/)
  pg = chain.postgres
  db = chain.db
@@ -1431,7 +1431,7 @@ func TestUpdateGlobalParams(t *testing.T) {
  txn := _assembleBasicTransferTxnFullySigned(t, chain, 200, 200, m0Pub, moneyPkString, m0Priv, mempool)
  txn.TxnNonce.ExpirationBlockHeight = uint64(chain.blockTip().Height + 1 + 5001)
  _signTxn(t, txn, m0Priv)
- newMP := NewDeSoMempool(chain, 0, 0, "", true, "", "")
+ newMP := NewDeSoMempool(chain, 0, 0, "", true, "", "", true)
  _, _, err = newMP.TryAcceptTransaction(txn, false, false)
  require.Error(err)
  require.Contains(err.Error(), TxErrorNonceExpirationBlockHeightOffsetExceeded)

lib/block_view_types_test.go

@@ -263,7 +263,7 @@ func TestUtxoEntryEncodeDecode(t *testing.T) {
  newMP := NewDeSoMempool(chain, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  mempool.resetPool(newMP)
 
  // Validating the first Bitcoin burn transaction via a UtxoView should

lib/blockchain.go

@@ -528,9 +528,11 @@ func (bc *Blockchain) _initChain() error {
 
  if bc.postgres != nil {
  err = bc.postgres.InitGenesisBlock(bc.params, bc.db)
- } else {
- err = InitDbWithDeSoGenesisBlock(bc.params, bc.db, bc.eventManager, bc.snapshot)
+ if err != nil {
+ return errors.Wrapf(err, "_initChain: Problem initializing postgres with genesis block")
+ }
  }
+ err = InitDbWithDeSoGenesisBlock(bc.params, bc.db, bc.eventManager, bc.snapshot, bc.postgres)
  if err != nil {
  return errors.Wrapf(err, "_initChain: Problem initializing db with genesis block")
  }

lib/blockchain_test.go

@@ -184,7 +184,7 @@ func AppendToMemLog(t *testing.T, prefix string) {
  f, err := os.OpenFile("mem.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
  if err == nil {
  defer f.Close()
- if _, err := f.WriteString(fmt.Sprintf("%s\t%s\tMemory Usage\t%v\n", prefix, t.Name(), float64(mem.Alloc)/float64(1e9))); err != nil {
+ if _, err := f.WriteString(fmt.Sprintf("%s\t%s\tMemory Usage\t%v\tTotal Alloc\t%v\n", prefix, t.Name(), float64(mem.Alloc)/float64(1e9), float64(mem.TotalAlloc)/float64(1e9))); err != nil {
  log.Println(err)
  }
  }
@@ -205,11 +205,11 @@ func NewLowDifficultyBlockchainWithParams(t *testing.T, params *DeSoParams) (
  // Set the number of txns per view regeneration to one while creating the txns
  ReadOnlyUtxoViewRegenerationIntervalTxns = 1
 
- chain, params, _ := NewLowDifficultyBlockchainWithParamsAndDb(t, params, len(os.Getenv("POSTGRES_URI")) > 0, 0)
+ chain, params, _ := NewLowDifficultyBlockchainWithParamsAndDb(t, params, len(os.Getenv("POSTGRES_URI")) > 0, 0, false)
  return chain, params, chain.db
 }
 
-func NewLowDifficultyBlockchainWithParamsAndDb(t *testing.T, params *DeSoParams, usePostgres bool, postgresPort uint32) (
+func NewLowDifficultyBlockchainWithParamsAndDb(t *testing.T, params *DeSoParams, usePostgres bool, postgresPort uint32, useProvidedParams bool) (
  *Blockchain, *DeSoParams, *embeddedpostgres.EmbeddedPostgres) {
  TestDeSoEncoderSetup(t)
  AppendToMemLog(t, "START")
@@ -237,13 +237,16 @@ func NewLowDifficultyBlockchainWithParamsAndDb(t *testing.T, params *DeSoParams,
  }
 
  timesource := chainlib.NewMedianTime()
- testParams := NewTestParams(params)
+ testParams := *params
+ if !useProvidedParams {
+ testParams = NewTestParams(params)
+ }
 
  // Temporarily modify the seed balances to make a specific public
  // key have some DeSo
  var snap *Snapshot
  if !usePostgres {
- snap, err, _ = NewSnapshot(db, dbDir, SnapshotBlockHeightPeriod, false, false, &testParams, false)
+ snap, err, _ = NewSnapshot(db, dbDir, SnapshotBlockHeightPeriod, false, false, &testParams, false, true)
  if err != nil {
  log.Fatal(err)
  }
@@ -260,6 +263,12 @@ func NewLowDifficultyBlockchainWithParamsAndDb(t *testing.T, params *DeSoParams,
  snap.Stop()
  CleanUpBadger(snap.SnapshotDb)
  }
+ if embpg != nil {
+ err = embpg.Stop()
+ if err != nil {
+ glog.Errorf("Error stopping embedded pg: %v", err)
+ }
+ }
  CleanUpBadger(db)
  TestDeSoEncoderShutdown(t)
  AppendToMemLog(t, "CLEANUP_END")
@@ -324,7 +333,7 @@ func NewTestMiner(t *testing.T, chain *Blockchain, params *DeSoParams, isSender
  mempool := NewDeSoMempool(
  chain, 0, /* rateLimitFeeRateNanosPerKB */
  0 /* minFeeRateNanosPerKB */, "", true,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  minerPubKeys := []string{}
  if isSender {
  minerPubKeys = append(minerPubKeys, senderPkString)
@@ -1742,3 +1751,16 @@ func TestForbiddenBlockSignaturePubKey(t *testing.T) {
  require.Error(err)
  require.Contains(err.Error(), RuleErrorForbiddenBlockProducerPublicKey)
 }
+
+func TestPGGenesisBlock(t *testing.T) {
+ // We skip this test in buildkite CI, but include it in GH actions postgres testing.
+ // Comment out this conditional to test locally.
+ if len(os.Getenv("POSTGRES_URI")) == 0 {
+ return
+ }
+ chain, params, _ := NewLowDifficultyBlockchainWithParamsAndDb(t, &DeSoTestnetParams, true, 5435, true)
+ for _, seedBalance := range params.SeedBalances {
+ bal := chain.postgres.GetBalance(NewPublicKey(seedBalance.PublicKey))
+ require.Equal(t, bal, seedBalance.AmountNanos)
+ }
+}

lib/db_utils.go

@@ -5096,7 +5096,7 @@ func DbBulkDeleteHeightHashToNodeInfo(handle *badger.DB, snap *Snapshot,
 // InitDbWithGenesisBlock initializes the database to contain only the genesis
 // block.
 func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB,
- eventManager *EventManager, snap *Snapshot) error {
+ eventManager *EventManager, snap *Snapshot, postgres *Postgres) error {
  // Construct a node for the genesis block. Its height is zero and it has
  // no parents. Its difficulty should be set to the initial
  // difficulty specified in the parameters and it should be assumed to be
@@ -5154,7 +5154,7 @@ func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB,
  // think things are initialized because we set the best block hash at the
  // top. We should fix this at some point so that an error in this step
  // wipes out the best hash.
- utxoView, err := NewUtxoView(handle, params, nil, snap)
+ utxoView, err := NewUtxoView(handle, params, postgres, snap)
  if err != nil {
  return fmt.Errorf(
  "InitDbWithDeSoGenesisBlock: Error initializing UtxoView")
@@ -5224,7 +5224,6 @@ func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB,
  UtxoOps: utxoOpsForBlock,
  })
  }
-
  // Flush all the data in the view.
  err = utxoView.FlushToDb(0)
  if err != nil {
@@ -9849,6 +9848,10 @@ func PerformanceBadgerOptions(dir string) badger.Options {
  return opts
 }
 
+func DefaultBadgerOptions(dir string) badger.Options {
+ return badger.DefaultOptions(dir)
+}
+
 // ---------------------------------------------
 // Associations
 // ---------------------------------------------

lib/db_utils_test.go

@@ -77,7 +77,7 @@ func GetTestBadgerDb() (_db *badger.DB, _dir string) {
  }
 
  // Open a badgerdb in a temporary directory.
- opts := PerformanceBadgerOptions(dir)
+ opts := DefaultBadgerOptions(dir)
  opts.Dir = dir
  opts.ValueDir = dir
  // Turn off logging for tests.
@@ -196,7 +196,7 @@ func TestInitDbWithGenesisBlock(t *testing.T) {
  db, _ := GetTestBadgerDb()
  defer CleanUpBadger(db)
 
- err := InitDbWithDeSoGenesisBlock(&DeSoTestnetParams, db, nil, nil)
+ err := InitDbWithDeSoGenesisBlock(&DeSoTestnetParams, db, nil, nil, nil)
  require.NoError(err)
 
  // Check the block index.

lib/load_test.go

@@ -151,7 +151,7 @@ func TestComputeMaxTPS(t *testing.T) {
  newMP := NewDeSoMempool(mempool.bc, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  mempool.resetPool(newMP)
  {
  timeStart := time.Now()

lib/mempool.go

@@ -255,6 +255,17 @@ type DeSoMempool struct {
  // We pass a copy of the data dir flag to the tx pool so that we can instantiate
  // temp badger db instances and dump mempool txns to them.
  dataDir string
+
+ // set to true for tests. This lowers the memory requirements of the mempool
+ // by using DefaultBadgerOptions instead of PerformanceBadgerOptions.
+ useDefaultBadgerOptions bool
+}
+
+func (mp *DeSoMempool) getBadgerOptions(dir string) badger.Options {
+ if mp.useDefaultBadgerOptions {
+ return DefaultBadgerOptions(dir)
+ }
+ return PerformanceBadgerOptions(dir)
 }
 
 // See comment on RemoveUnconnectedTxn. The mempool lock must be called for writing
@@ -321,6 +332,7 @@ func (mp *DeSoMempool) resetPool(newPool *DeSoMempool) {
  mp.backupUniversalUtxoView = newPool.backupUniversalUtxoView
  mp.universalUtxoView = newPool.universalUtxoView
  mp.universalTransactionList = newPool.universalTransactionList
+ mp.useDefaultBadgerOptions = newPool.useDefaultBadgerOptions
 
  // We don't adjust blockCypherAPIKey or blockCypherCheckDoubleSpendChan
  // since those should be unaffected
@@ -381,7 +393,7 @@ func (mp *DeSoMempool) UpdateAfterConnectBlock(blk *MsgDeSoBlock) (_txnsAddedToM
  0, /* minFeeRateNanosPerKB */
  "", /*blockCypherAPIKey*/
  false, /*runReadOnlyViewUpdater*/
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", mp.useDefaultBadgerOptions)
 
  // Get all the transactions from the old pool object.
  oldMempoolTxns, oldUnconnectedTxns, err := mp._getTransactionsOrderedByTimeAdded()
@@ -485,7 +497,7 @@ func (mp *DeSoMempool) UpdateAfterDisconnectBlock(blk *MsgDeSoBlock) {
  newPool := NewDeSoMempool(mp.bc, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", mp.useDefaultBadgerOptions)
 
  // Add the transactions from the block to the new pool (except for the block reward,
  // which should always be the first transaction). Break out if we encounter
@@ -767,7 +779,7 @@ func (mp *DeSoMempool) OpenTempDBAndDumpTxns() error {
  if err != nil {
  return fmt.Errorf("OpenTempDBAndDumpTxns: Error making top-level dir: %v", err)
  }
- tempMempoolDBOpts := PerformanceBadgerOptions(tempMempoolDBDir)
+ tempMempoolDBOpts := mp.getBadgerOptions(tempMempoolDBDir)
  tempMempoolDBOpts.ValueDir = tempMempoolDBDir
  tempMempoolDB, err := badger.Open(tempMempoolDBOpts)
  if err != nil {
@@ -2388,7 +2400,7 @@ func (mp *DeSoMempool) inefficientRemoveTransaction(tx *MsgDeSoTxn) {
  newPool := NewDeSoMempool(mp.bc, 0, /* rateLimitFeeRateNanosPerKB */
  0, /* minFeeRateNanosPerKB */
  "" /*blockCypherAPIKey*/, false,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", mp.useDefaultBadgerOptions)
  // At this point the block txns have been added to the new pool. Now we need to
  // add the txns from the original pool. Start by fetching them in slice form.
  oldMempoolTxns, oldUnconnectedTxns, err := mp._getTransactionsOrderedByTimeAdded()
@@ -2562,7 +2574,7 @@ func (mp *DeSoMempool) LoadTxnsFromDB() {
  }
 
  // If we make it this far, we found a mempool dump to load. Woohoo!
- tempMempoolDBOpts := PerformanceBadgerOptions(savedTxnsDir)
+ tempMempoolDBOpts := mp.getBadgerOptions(savedTxnsDir)
  tempMempoolDBOpts.ValueDir = savedTxnsDir
  glog.Infof("LoadTxnsFrom: Opening new temp db %v", savedTxnsDir)
  tempMempoolDB, err := badger.Open(tempMempoolDBOpts)
@@ -2600,7 +2612,7 @@ func (mp *DeSoMempool) Stop() {
 // Create a new pool with no transactions in it.
 func NewDeSoMempool(_bc *Blockchain, _rateLimitFeerateNanosPerKB uint64,
  _minFeerateNanosPerKB uint64, _blockCypherAPIKey string,
- _runReadOnlyViewUpdater bool, _dataDir string, _mempoolDumpDir string) *DeSoMempool {
+ _runReadOnlyViewUpdater bool, _dataDir string, _mempoolDumpDir string, useDefaultBadgerOptions bool) *DeSoMempool {
 
  utxoView, _ := NewUtxoView(_bc.db, _bc.params, _bc.postgres, _bc.snapshot)
  backupUtxoView, _ := NewUtxoView(_bc.db, _bc.params, _bc.postgres, _bc.snapshot)
@@ -2624,6 +2636,7 @@ func NewDeSoMempool(_bc *Blockchain, _rateLimitFeerateNanosPerKB uint64,
  readOnlyUniversalTransactionMap: make(map[BlockHash]*MempoolTx),
  readOnlyOutpoints: make(map[UtxoKey]*MsgDeSoTxn),
  dataDir: _dataDir,
+ useDefaultBadgerOptions: useDefaultBadgerOptions,
  }
 
  if newPool.mempoolDir != "" {

lib/mempool_test.go

@@ -54,7 +54,7 @@ func TestMempoolLongChainOfDependencies(t *testing.T) {
  mp := NewDeSoMempool(
  chain, 0, /* rateLimitFeeRateNanosPerKB */
  0 /* minFeeRateNanosPerKB */, "", true,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  _, err := mp.processTransaction(txn1, false /*allowUnconnectedTxn*/, false /*rateLimit*/, 0 /*peerID*/, true /*verifySignatures*/)
  require.NoError(err)
 
@@ -118,7 +118,7 @@ func TestMempoolRateLimit(t *testing.T) {
  mpNoMinFees := NewDeSoMempool(
  chain, 0, /* rateLimitFeeRateNanosPerKB */
  0 /* minFeeRateNanosPerKB */, "", true,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
 
  // Create a transaction that sends 1 DeSo to the recipient as its
  // zeroth output.
@@ -134,7 +134,7 @@ func TestMempoolRateLimit(t *testing.T) {
  mpWithMinFee := NewDeSoMempool(
  chain, 0, /* rateLimitFeeRateNanosPerKB */
  100 /* minFeeRateNanosPerKB */, "", true,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  _, err = mpWithMinFee.processTransaction(txn1, false /*allowUnconnectedTxn*/, true /*rateLimit*/, 0 /*peerID*/, false /*verifySignatures*/)
  require.Error(err)
  require.Contains(err.Error(), TxErrorInsufficientFeeMinFee)
@@ -183,7 +183,7 @@ func TestMempoolRateLimit(t *testing.T) {
  mpWithRateLimit := NewDeSoMempool(
  chain, 100, /* rateLimitFeeRateNanosPerKB */
  0 /* minFeeRateNanosPerKB */, "", true,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
  processingErrors := []error{}
  for _, txn := range txnsCreated {
  _, err := mpWithRateLimit.processTransaction(txn, false /*allowUnconnectedTxn*/, true /*rateLimit*/, 0 /*peerID*/, false /*verifySignatures*/)
@@ -311,7 +311,7 @@ func TestMempoolAugmentedUtxoViewTransactionChain(t *testing.T) {
  mp := NewDeSoMempool(
  chain, 0, /* rateLimitFeeRateNanosPerKB */
  0 /* minFeeRateNanosPerKB */, "", true,
- "" /*dataDir*/, "")
+ "" /*dataDir*/, "", true)
 
  // Process the first transaction.
  mempoolTx1, err := mp.processTransaction(txn1, false /*allowUnconnectedTxn*/, false /*rateLimit*/, 0 /*peerID*/, true /*verifySignatures*/)