services/horizon: Add transaction set operation count to history_ledg…

…er. (#2690) ### What Extend ingestion to store the total number of operations in the transaction set and expose it in the ledger resource via `tx_set_operation_count`. ### Why This feature was requested in #1385 and it will help us assess the used capacity of a transaction set. With this change we have now the total count of operations in the transaction set and the number of successful operations. Fix #1385.
stellar · Jun 16, 2020 · dce848d · dce848d
1 parent 98da9ed
commit dce848d
Show file tree

Hide file tree

Showing 25 changed files with 196 additions and 131 deletions.
diff --git a/protocols/horizon/main.go b/protocols/horizon/main.go
@@ -219,6 +219,7 @@ type Ledger struct {
 	SuccessfulTransactionCount int32     `json:"successful_transaction_count"`
 	FailedTransactionCount     *int32    `json:"failed_transaction_count"`
 	OperationCount             int32     `json:"operation_count"`
+	TxSetOperationCount        *int32    `json:"tx_set_operation_count"`
 	ClosedAt                   time.Time `json:"closed_at"`
 	TotalCoins                 string    `json:"total_coins"`
 	FeePool                    string    `json:"fee_pool"`

diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md
@@ -1,7 +1,12 @@
 # Changelog
 
 All notable changes to this project will be documented in this
-file. This project adheres to [Semantic Versioning](http://semver.org/).
+file. This project adheres to [Semantic Versioning](http://semver.org/).x
+
+## Unreleased
+
+* Add transaction set operation count to `history_ledger`([#2690](https://github.com/stellar/go/pull/2690)).
+Extend ingestion to store the total number of operations in the transaction set and expose it in the ledger resource via `tx_set_operation_count`. This feature allow you to assess the used capacity of a transaction set.
 
 ## v1.4.0
 
@@ -60,8 +65,8 @@ The changes are required by [CAP-15](https://github.com/stellar/stellar-protocol
 
 * Added support for [CAP-27](https://github.com/stellar/stellar-protocol/blob/master/core/cap-0027.md) and [SEP-23](https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0023.md) [#2491](https://github.com/stellar/go/pull/2491).
 * The XDR definition of a transaction memo is a string.
-However, XDR strings are actually binary blobs with no enforced encoding. 
-It is possible to set the memo in a transaction envelope to a binary sequence which is not valid ASCII or unicode. 
+However, XDR strings are actually binary blobs with no enforced encoding.
+It is possible to set the memo in a transaction envelope to a binary sequence which is not valid ASCII or unicode.
 Previously, if you wanted to recover the original binary sequence for a transaction memo, you would have to decode the transaction's envelope.
 In this release, we have added a `memo_bytes` field to the Horizon transaction response for transactions with `memo_type` equal `text`.
 `memo_bytes` stores the base 64 encoding of the memo bytes set in the transaction envelope [#2485](https://github.com/stellar/go/pull/2485).
@@ -145,7 +150,7 @@ To execute the migration run `horizon db migrate up` using the Horizon v1.1.0 bi
       "trustor": "GA332TXN6BX2DYKGYB7FW5BWV2JLQKERNX4T7EUJT4MHWOW2TSGC2SPM",
       "asset_type": "credit_alphanum4",
       "asset_code": "USD"
-    }    
+    }
     </pre>
 * It is no longer possible to use Redis as a mechanism for rate-limiting requests ([#2409](https://github.com/stellar/go/pull/2409)).
 

diff --git a/services/horizon/internal/actions/account_test.go b/services/horizon/internal/actions/account_test.go
@@ -218,7 +218,7 @@ func TestAccountInfo(t *testing.T) {
 				CloseTime: xdr.TimePoint(ledgerFourCloseTime),
 			},
 		},
-	}, 0, 0, 0, 0)
+	}, 0, 0, 0, 0, 0)
 	assert.NoError(t, err)
 
 	account, err := AccountInfo(tt.Ctx, &history.Q{tt.HorizonSession()}, accountID.Address())
@@ -376,7 +376,7 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) {
 				CloseTime: xdr.TimePoint(ledgerCloseTime),
 			},
 		},
-	}, 0, 0, 0, 0)
+	}, 0, 0, 0, 0, 0)
 	assert.NoError(t, err)
 
 	for _, row := range accountSigners {

diff --git a/services/horizon/internal/actions/offer_test.go b/services/horizon/internal/actions/offer_test.go
@@ -78,7 +78,7 @@ func TestGetOfferByIDHandler(t *testing.T) {
 				CloseTime: xdr.TimePoint(ledgerCloseTime),
 			},
 		},
-	}, 0, 0, 0, 0)
+	}, 0, 0, 0, 0, 0)
 	tt.Assert.NoError(err)
 
 	batch := q.NewOffersBatchInsertBuilder(0)
@@ -188,7 +188,7 @@ func TestGetOffersHandler(t *testing.T) {
 				CloseTime: xdr.TimePoint(ledgerCloseTime),
 			},
 		},
-	}, 0, 0, 0, 0)
+	}, 0, 0, 0, 0, 0)
 	tt.Assert.NoError(err)
 
 	batch := q.NewOffersBatchInsertBuilder(0)

diff --git a/services/horizon/internal/actions_account_test.go b/services/horizon/internal/actions_account_test.go
@@ -22,7 +22,7 @@ func TestAccountActions_InvalidID(t *testing.T) {
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 100,
 		},
-	}, 0, 0, 0, 0)
+	}, 0, 0, 0, 0, 0)
 	ht.Assert.NoError(err)
 
 	// existing account

diff --git a/services/horizon/internal/actions_data_test.go b/services/horizon/internal/actions_data_test.go
@@ -42,7 +42,7 @@ func TestDataActions_Show(t *testing.T) {
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 100,
 		},
-	}, 0, 0, 0, 0)
+	}, 0, 0, 0, 0, 0)
 	ht.Assert.NoError(err)
 
 	rows, err := q.InsertAccountData(data1, 1234)

diff --git a/services/horizon/internal/actions_ledger_test.go b/services/horizon/internal/actions_ledger_test.go
@@ -39,13 +39,14 @@ func TestLedgerActions_Show(t *testing.T) {
 		ht.Assert.NotEmpty(result.HeaderXDR)
 		ht.Assert.Equal(int32(3), result.SuccessfulTransactionCount)
 		ht.Assert.Equal(int32(0), *result.FailedTransactionCount)
+		ht.Assert.Nil(result.TxSetOperationCount)
 	}
 
 	// There's no way to test previous versions of ingestion right now
 	// so let's manually update the state to look like version 14 of ingesiton
 	// only the latest gap is considered for determining the elder ledger
 	_, err = ht.HorizonDB.Exec(`
-		UPDATE history_ledgers SET successful_transaction_count = NULL, failed_transaction_count = NULL WHERE sequence = 2
+		UPDATE history_ledgers SET successful_transaction_count = NULL, failed_transaction_count = NULL, tx_set_operation_count = 5 WHERE sequence = 2
 	`)
 	ht.Require.NoError(err, "failed to update history_ledgers")
 
@@ -59,6 +60,7 @@ func TestLedgerActions_Show(t *testing.T) {
 		ht.Assert.NotEmpty(result.HeaderXDR)
 		ht.Assert.Equal(int32(3), result.SuccessfulTransactionCount)
 		ht.Assert.Nil(result.FailedTransactionCount)
+		ht.Assert.Equal(int32(5), *result.TxSetOperationCount)
 	}
 
 	// ledger higher than history

diff --git a/services/horizon/internal/db2/history/ledger.go b/services/horizon/internal/db2/history/ledger.go
@@ -92,6 +92,7 @@ type QLedgers interface {
 		successTxsCount int,
 		failedTxsCount int,
 		opCount int,
+		txSetOpCount int,
 		ingestVersion int,
 	) (int64, error)
 }
@@ -103,13 +104,15 @@ func (q *Q) InsertLedger(
 	successTxsCount int,
 	failedTxsCount int,
 	opCount int,
+	txSetOpCount int,
 	ingestVersion int,
 ) (int64, error) {
 	m, err := ledgerHeaderToMap(
 		ledger,
 		successTxsCount,
 		failedTxsCount,
 		opCount,
+		txSetOpCount,
 		ingestVersion,
 	)
 	if err != nil {
@@ -130,6 +133,7 @@ func ledgerHeaderToMap(
 	successTxsCount int,
 	failedTxsCount int,
 	opCount int,
+	txSetOpCount int,
 	importerVersion int,
 ) (map[string]interface{}, error) {
 	ledgerHeaderBase64, err := xdr.MarshalBase64(ledger.Header)
@@ -155,6 +159,7 @@ func ledgerHeaderToMap(
 		"successful_transaction_count": successTxsCount,
 		"failed_transaction_count":     failedTxsCount,
 		"operation_count":              opCount,
+		"tx_set_operation_count":       txSetOpCount,
 		"protocol_version":             ledger.Header.LedgerVersion,
 		"ledger_header":                ledgerHeaderBase64,
 	}, nil
@@ -170,6 +175,7 @@ var selectLedger = sq.Select(
 	"hl.successful_transaction_count",
 	"hl.failed_transaction_count",
 	"hl.operation_count",
+	"hl.tx_set_operation_count",
 	"hl.closed_at",
 	"hl.created_at",
 	"hl.updated_at",

diff --git a/services/horizon/internal/db2/history/ledger_test.go b/services/horizon/internal/db2/history/ledger_test.go
@@ -65,6 +65,7 @@ func TestInsertLedger(t *testing.T) {
 		TransactionCount:           12,
 		SuccessfulTransactionCount: new(int32),
 		FailedTransactionCount:     new(int32),
+		TxSetOperationCount:        new(int32),
 		OperationCount:             23,
 		TotalCoins:                 23451,
 		FeePool:                    213,
@@ -76,6 +77,7 @@ func TestInsertLedger(t *testing.T) {
 	}
 	*expectedLedger.SuccessfulTransactionCount = 12
 	*expectedLedger.FailedTransactionCount = 3
+	*expectedLedger.TxSetOperationCount = 26
 
 	var ledgerHash, previousLedgerHash xdr.Hash
 
@@ -112,6 +114,7 @@ func TestInsertLedger(t *testing.T) {
 		12,
 		3,
 		23,
+		26,
 		int(expectedLedger.ImporterVersion),
 	)
 	tt.Assert.NoError(err)

diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go
@@ -395,6 +395,7 @@ type Ledger struct {
 	SuccessfulTransactionCount *int32      `db:"successful_transaction_count"`
 	FailedTransactionCount     *int32      `db:"failed_transaction_count"`
 	OperationCount             int32       `db:"operation_count"`
+	TxSetOperationCount        *int32      `db:"tx_set_operation_count"`
 	ClosedAt                   time.Time   `db:"closed_at"`
 	CreatedAt                  time.Time   `db:"created_at"`
 	UpdatedAt                  time.Time   `db:"updated_at"`

diff --git a/services/horizon/internal/db2/history/mock_q_ledgers.go b/services/horizon/internal/db2/history/mock_q_ledgers.go
@@ -14,8 +14,9 @@ func (m *MockQLedgers) InsertLedger(
 	successTxsCount int,
 	failedTxsCount int,
 	opCount int,
+	txSetOpCount int,
 	ingestVersion int,
 ) (int64, error) {
-	a := m.Called(ledger, successTxsCount, failedTxsCount, opCount, ingestVersion)
+	a := m.Called(ledger, successTxsCount, failedTxsCount, opCount, txSetOpCount, ingestVersion)
 	return a.Get(0).(int64), a.Error(1)
 }
diff --git a/services/horizon/internal/db2/schema/bindata.go b/services/horizon/internal/db2/schema/bindata.go
diff --git a/services/horizon/internal/db2/schema/migrations/37_add_tx_set_operation_count_to_ledgers.sql b/services/horizon/internal/db2/schema/migrations/37_add_tx_set_operation_count_to_ledgers.sql
@@ -0,0 +1,7 @@
+-- +migrate Up
+
+ALTER TABLE history_ledgers ADD tx_set_operation_count integer DEFAULT NULL;
+
+-- +migrate Down
+
+ALTER TABLE history_ledgers DROP COLUMN tx_set_operation_count;
diff --git a/services/horizon/internal/expingest/processor_runner_test.go b/services/horizon/internal/expingest/processor_runner_test.go
@@ -327,7 +327,7 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
 		Return(mockTransactionsBatchInsertBuilder).Twice()
 
-	q.MockQLedgers.On("InsertLedger", ledger, 0, 0, 0, CurrentVersion).
+	q.MockQLedgers.On("InsertLedger", ledger, 0, 0, 0, 0, CurrentVersion).
 		Return(int64(1), nil).Once()
 
 	runner := ProcessorRunner{

diff --git a/services/horizon/internal/expingest/processors/ledgers_processor.go b/services/horizon/internal/expingest/processors/ledgers_processor.go
@@ -14,6 +14,7 @@ type LedgersProcessor struct {
 	successTxCount int
 	failedTxCount  int
 	opCount        int
+	txSetOpCount   int
 }
 
 func NewLedgerProcessor(
@@ -29,9 +30,11 @@ func NewLedgerProcessor(
 }
 
 func (p *LedgersProcessor) ProcessTransaction(transaction io.LedgerTransaction) (err error) {
+	opCount := len(transaction.Envelope.Operations())
+	p.txSetOpCount += opCount
 	if transaction.Result.Successful() {
 		p.successTxCount++
-		p.opCount += len(transaction.Envelope.Operations())
+		p.opCount += opCount
 	} else {
 		p.failedTxCount++
 	}
@@ -45,6 +48,7 @@ func (p *LedgersProcessor) Commit() error {
 		p.successTxCount,
 		p.failedTxCount,
 		p.opCount,
+		p.txSetOpCount,
 		p.ingestVersion,
 	)
 

diff --git a/services/horizon/internal/expingest/processors/ledgers_processor_test.go b/services/horizon/internal/expingest/processors/ledgers_processor_test.go
@@ -21,6 +21,7 @@ type LedgersProcessorTestSuiteLedger struct {
 	opCount       int
 	ingestVersion int
 	txs           []io.LedgerTransaction
+	txSetOpCount  int
 }
 
 func TestLedgersProcessorTestSuiteLedger(t *testing.T) {
@@ -87,6 +88,7 @@ func (s *LedgersProcessorTestSuiteLedger) SetupTest() {
 	s.successCount = 2
 	s.failedCount = 1
 	s.opCount = 5
+	s.txSetOpCount = 8
 }
 
 func (s *LedgersProcessorTestSuiteLedger) TearDownTest() {
@@ -100,6 +102,7 @@ func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerSucceeds() {
 		s.successCount,
 		s.failedCount,
 		s.opCount,
+		s.txSetOpCount,
 		s.ingestVersion,
 	).Return(int64(1), nil)
 
@@ -120,6 +123,7 @@ func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerReturnsError() {
 		mock.Anything,
 		mock.Anything,
 		mock.Anything,
+		mock.Anything,
 	).Return(int64(0), errors.New("transient error"))
 
 	err := s.processor.Commit()
@@ -135,6 +139,7 @@ func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerNoRowsAffected() {
 		mock.Anything,
 		mock.Anything,
 		mock.Anything,
+		mock.Anything,
 	).Return(int64(0), nil)
 
 	err := s.processor.Commit()

diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go
@@ -260,7 +260,7 @@ func TestStateMiddleware(t *testing.T) {
 					LedgerSeq:          testCase.latestHistoryLedger,
 					PreviousLedgerHash: xdr.Hash{byte(i)},
 				},
-			}, 0, 0, 0, 0)
+			}, 0, 0, 0, 0, 0)
 			tt.Assert.NoError(err)
 			tt.Assert.NoError(q.UpdateLastLedgerExpIngest(testCase.lastIngestedLedger))
 			tt.Assert.NoError(q.UpdateExpIngestVersion(testCase.ingestionVersion))

diff --git a/services/horizon/internal/resourceadapter/ledger.go b/services/horizon/internal/resourceadapter/ledger.go
@@ -25,6 +25,7 @@ func PopulateLedger(ctx context.Context, dest *protocol.Ledger, row history.Ledg
 	}
 	dest.FailedTransactionCount = row.FailedTransactionCount
 	dest.OperationCount = row.OperationCount
+	dest.TxSetOperationCount = row.TxSetOperationCount
 	dest.ClosedAt = row.ClosedAt
 	dest.TotalCoins = amount.String(xdr.Int64(row.TotalCoins))
 	dest.FeePool = amount.String(xdr.Int64(row.FeePool))