services/horizon: Remove ingest failed transactions flag (#2702)

Remove ingest failed transactions flag so that Horizon will always ingest failed transactions
stellar · Jun 18, 2020 · 776aa42 · 776aa42
1 parent c157ce9
commit 776aa42
Show file tree

Hide file tree

Showing 35 changed files with 44 additions and 1,630 deletions.
diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md
@@ -7,6 +7,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).x
 
 * Add transaction set operation count to `history_ledger`([#2690](https://github.com/stellar/go/pull/2690)).
 Extend ingestion to store the total number of operations in the transaction set and expose it in the ledger resource via `tx_set_operation_count`. This feature allow you to assess the used capacity of a transaction set.
+* Remove `--ingest-failed-transactions` flag. From now on Horizon will always ingest failed transactions. WARNING: if your application is using Horizon DB directly (not recommended!) remember that now it will also contain failed txs. ([#2702](https://github.com/stellar/go/pull/2702)).
 
 ## v1.4.0
 

diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go
@@ -173,11 +173,10 @@ var dbReingestRangeCmd = &cobra.Command{
 		}
 
 		ingestConfig := expingest.Config{
-			CoreSession:              coreSession,
-			NetworkPassphrase:        config.NetworkPassphrase,
-			HistorySession:           horizonSession,
-			HistoryArchiveURL:        config.HistoryArchiveURLs[0],
-			IngestFailedTransactions: config.IngestFailedTransactions,
+			CoreSession:       coreSession,
+			NetworkPassphrase: config.NetworkPassphrase,
+			HistorySession:    horizonSession,
+			HistoryArchiveURL: config.HistoryArchiveURLs[0],
 		}
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.StellarCorePath = config.StellarCoreBinaryPath

diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go
@@ -102,11 +102,10 @@ var ingestVerifyRangeCmd = &cobra.Command{
 		}
 
 		ingestConfig := expingest.Config{
-			CoreSession:              coreSession,
-			NetworkPassphrase:        config.NetworkPassphrase,
-			HistorySession:           horizonSession,
-			HistoryArchiveURL:        config.HistoryArchiveURLs[0],
-			IngestFailedTransactions: config.IngestFailedTransactions,
+			CoreSession:       coreSession,
+			NetworkPassphrase: config.NetworkPassphrase,
+			HistorySession:    horizonSession,
+			HistoryArchiveURL: config.HistoryArchiveURLs[0],
 		}
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.StellarCorePath = config.StellarCoreBinaryPath
@@ -182,11 +181,10 @@ var ingestStressTestCmd = &cobra.Command{
 		}
 
 		ingestConfig := expingest.Config{
-			CoreSession:              coreSession,
-			NetworkPassphrase:        config.NetworkPassphrase,
-			HistorySession:           horizonSession,
-			HistoryArchiveURL:        config.HistoryArchiveURLs[0],
-			IngestFailedTransactions: config.IngestFailedTransactions,
+			CoreSession:       coreSession,
+			NetworkPassphrase: config.NetworkPassphrase,
+			HistorySession:    horizonSession,
+			HistoryArchiveURL: config.HistoryArchiveURLs[0],
 		}
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.StellarCorePath = config.StellarCoreBinaryPath

diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go
@@ -329,13 +329,6 @@ var configOpts = support.ConfigOptions{
 		FlagDefault: false,
 		Usage:       "causes this horizon process to ingest data from stellar-core into horizon's db",
 	},
-	&support.ConfigOption{
-		Name:        "ingest-failed-transactions",
-		ConfigKey:   &config.IngestFailedTransactions,
-		OptType:     types.Bool,
-		FlagDefault: false,
-		Usage:       "causes this horizon process to ingest failed transactions data",
-	},
 	&support.ConfigOption{
 		Name:        "cursor-name",
 		EnvVar:      "CURSOR_NAME",

diff --git a/services/horizon/internal/action_test.go b/services/horizon/internal/action_test.go
@@ -22,7 +22,7 @@ func TestGetTransactionPage(t *testing.T) {
 	defer tt.Finish()
 
 	ctx := context.Background()
-	w := mustInitWeb(ctx, &history.Q{tt.HorizonSession()}, time.Duration(5), 0, true)
+	w := mustInitWeb(ctx, &history.Q{tt.HorizonSession()}, time.Duration(5), 0)
 
 	// filter by account
 	params := &indexActionQueryParams{

diff --git a/services/horizon/internal/actions/operation.go b/services/horizon/internal/actions/operation.go
@@ -50,8 +50,7 @@ func (qp OperationsQuery) Validate() error {
 
 // GetOperationsHandler is the action handler for all end-points returning a list of operations.
 type GetOperationsHandler struct {
-	OnlyPayments                bool
-	IngestingFailedTransactions bool
+	OnlyPayments bool
 }
 
 // GetResourcePage returns a page of operations.
@@ -74,12 +73,6 @@ func (handler GetOperationsHandler) GetResourcePage(w HeaderWriter, r *http.Requ
 		return nil, err
 	}
 
-	if qp.IncludeFailedTransactions && !handler.IngestingFailedTransactions {
-		err = errors.New("`include_failed` parameter is unavailable when Horizon is not ingesting failed " +
-			"transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.")
-		return nil, problem.MakeInvalidFieldProblem("include_failed", err)
-	}
-
 	historyQ, err := HistoryQFromRequest(r)
 	if err != nil {
 		return nil, err

diff --git a/services/horizon/internal/actions/operation_test.go b/services/horizon/internal/actions/operation_test.go
@@ -19,7 +19,7 @@ func TestGetOperationsWithoutFilter(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	records, err := handler.GetResourcePage(
 		httptest.NewRecorder(),
@@ -37,7 +37,7 @@ func TestGetOperationsExclusiveFilters(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	testCases := []struct {
 		desc  string
@@ -92,7 +92,7 @@ func TestGetOperationsFilterByAccountID(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	testCases := []struct {
 		accountID string
@@ -133,7 +133,7 @@ func TestGetOperationsFilterByTxID(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	testCases := []struct {
 		desc          string
@@ -207,7 +207,7 @@ func TestGetOperationsIncludeFailed(t *testing.T) {
 	tt.Scenario("failed_transactions")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	records, err := handler.GetResourcePage(
 		httptest.NewRecorder(),
@@ -329,28 +329,6 @@ func TestGetOperationsIncludeFailed(t *testing.T) {
 		"Filter should be true or false",
 		p.Extras["reason"],
 	)
-
-	handler = GetOperationsHandler{
-		IngestingFailedTransactions: false,
-	}
-
-	_, err = handler.GetResourcePage(
-		httptest.NewRecorder(),
-		makeRequest(
-			t, map[string]string{
-				"include_failed": "true",
-			}, map[string]string{}, q.Session,
-		),
-	)
-	tt.Assert.Error(err)
-	tt.Assert.IsType(&problem.P{}, err)
-	p = err.(*problem.P)
-	tt.Assert.Equal("bad_request", p.Type)
-	tt.Assert.Equal("include_failed", p.Extras["invalid_field"])
-	tt.Assert.Equal(
-		"`include_failed` parameter is unavailable when Horizon is not ingesting failed transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.",
-		p.Extras["reason"],
-	)
 }
 
 func TestGetOperationsFilterByLedgerID(t *testing.T) {
@@ -359,7 +337,7 @@ func TestGetOperationsFilterByLedgerID(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	testCases := []struct {
 		ledgerID    string
@@ -430,8 +408,7 @@ func TestGetOperationsOnlyPayments(t *testing.T) {
 
 	q := &history.Q{tt.HorizonSession()}
 	handler := GetOperationsHandler{
-		IngestingFailedTransactions: true,
-		OnlyPayments:                true,
+		OnlyPayments: true,
 	}
 
 	records, err := handler.GetResourcePage(
@@ -510,7 +487,7 @@ func TestOperation_CreatedAt(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	records, err := handler.GetResourcePage(
 		httptest.NewRecorder(),
@@ -535,7 +512,7 @@ func TestGetOperationsPagination(t *testing.T) {
 	tt.Scenario("base")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	records, err := handler.GetResourcePage(
 		httptest.NewRecorder(),
@@ -592,7 +569,7 @@ func TestGetOperations_IncludeTransactions(t *testing.T) {
 	tt.Scenario("failed_transactions")
 
 	q := &history.Q{tt.HorizonSession()}
-	handler := GetOperationsHandler{IngestingFailedTransactions: true}
+	handler := GetOperationsHandler{}
 
 	_, err := handler.GetResourcePage(
 		httptest.NewRecorder(),

diff --git a/services/horizon/internal/actions_operation_fee_stats.go b/services/horizon/internal/actions_operation_fee_stats.go
@@ -1,14 +1,12 @@
 package horizon
 
 import (
-	"net/http"
 	"strconv"
 
 	hProtocol "github.com/stellar/go/protocols/horizon"
 	"github.com/stellar/go/services/horizon/internal/actions"
 	"github.com/stellar/go/services/horizon/internal/operationfeestats"
 	"github.com/stellar/go/support/render/httpjson"
-	"github.com/stellar/go/support/render/problem"
 )
 
 // This file contains the actions:
@@ -26,20 +24,6 @@ type FeeStatsAction struct {
 
 // JSON is a method for actions.JSON
 func (action *FeeStatsAction) JSON() error {
-	if !action.App.config.IngestFailedTransactions {
-		// If Horizon is not ingesting failed transaction it does not make sense to display
-		// operation fee stats because they will be incorrect.
-		p := problem.P{
-			Type:   "endpoint_not_available",
-			Title:  "Endpoint Not Available",
-			Status: http.StatusNotImplemented,
-			Detail: "/fee_stats is unavailable when Horizon is not ingesting failed " +
-				"transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.",
-		}
-		problem.Render(action.R.Context(), action.W, p)
-		return nil
-	}
-
 	action.Do(
 		action.loadRecords,
 		func() {

diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go
@@ -466,7 +466,7 @@ func (a *App) init() {
 	a.reaper = reap.New(a.config.HistoryRetentionCount, a.HorizonSession(context.Background()))
 
 	// web.init
-	a.web = mustInitWeb(a.ctx, a.historyQ, a.config.SSEUpdateFrequency, a.config.StaleThreshold, a.config.IngestFailedTransactions)
+	a.web = mustInitWeb(a.ctx, a.historyQ, a.config.SSEUpdateFrequency, a.config.StaleThreshold)
 
 	// web.rate-limiter
 	a.web.rateLimiter = maybeInitWebRateLimiter(a.config.RateQuota)

diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go
@@ -45,8 +45,6 @@ type Config struct {
 	TLSKey string
 	// Ingest toggles whether this horizon instance should run the data ingestion subsystem.
 	Ingest bool
-	// IngestFailedTransactions toggles whether to ingest failed transactions
-	IngestFailedTransactions bool
 	// CursorName is the cursor used for ingesting from stellar-core.
 	// Setting multiple cursors in different Horizon instances allows multiple
 	// Horizons to ingest from the same stellar-core instance without cursor

diff --git a/services/horizon/internal/db2/core/account.go b/services/horizon/internal/db2/core/account.go