From 03dce0f19b707ca84b973c92d122b88dee881952 Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Mon, 29 Nov 2021 17:38:27 +0000
Subject: [PATCH 01/17] Skip in-memory flag for catchup with captive core

---
 ingest/ledgerbackend/stellar_core_runner.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index df9d8fc271..024120001d 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -264,7 +264,6 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 	r.cmd = r.createCmd(
 		"catchup", rangeArg,
 		"--metadata-output-stream", r.getPipeName(),
-		"--in-memory",
 	)
 
 	var err error

From 0426a4947c0ae96d5f408929f64d228ac5e0f03b Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Tue, 18 Jan 2022 17:00:54 +0000
Subject: [PATCH 02/17] run a catchup before run, intead of runFrom

---
 ingest/ledgerbackend/captive_core_backend.go  |  60 ++--
 .../captive_core_backend_test.go              | 274 +++++++++++-------
 ingest/ledgerbackend/stellar_core_runner.go   |   8 +-
 3 files changed, 194 insertions(+), 148 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 80890229f2..d4a46b7cd8 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -2,7 +2,6 @@ package ledgerbackend
 
 import (
 	"context"
-	"encoding/hex"
 	"os"
 	"sync"
 
@@ -248,20 +247,39 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	}
 
 	var runner stellarCoreRunnerInterface
-	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline); err != nil {
+	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOffline); err != nil {
 		return errors.Wrap(err, "error creating stellar-core runner")
-	} else {
-		// only assign c.stellarCoreRunner if runner is not nil to avoid nil interface check
-		// see https://golang.org/doc/faq#nil_error
-		c.stellarCoreRunner = runner
 	}
+	c.stellarCoreRunner = runner
 
-	runFrom, ledgerHash, err := c.runFromParams(ctx, from)
+	runFrom, err := c.runFromParams(ctx, from)
 	if err != nil {
 		return errors.Wrap(err, "error calculating ledger and hash for stellar-core run")
 	}
 
-	err = c.stellarCoreRunner.runFrom(runFrom, ledgerHash)
+	err = c.stellarCoreRunner.catchup(runFrom, runFrom-1)
+	if err != nil {
+		return errors.Wrap(err, "error running stellar-core")
+	}
+	for range c.stellarCoreRunner.getMetaPipe() {
+		// Drain the pipe
+	}
+	if err := c.stellarCoreRunner.close(); err != nil {
+		return errors.Wrap(err, "error running stellar-core")
+	}
+
+	if processExited, err := c.stellarCoreRunner.getProcessExitError(); err != nil {
+		return errors.Wrap(err, "error running stellar-core")
+	} else if !processExited {
+		return errors.New("error creating stellar-core runner")
+	}
+
+	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline); err != nil {
+		return errors.Wrap(err, "error creating stellar-core runner")
+	}
+	c.stellarCoreRunner = runner
+
+	err = c.stellarCoreRunner.run()
 	if err != nil {
 		return errors.Wrap(err, "error running stellar-core")
 	}
@@ -279,14 +297,13 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 }
 
 // runFromParams receives a ledger sequence and calculates the required values to call stellar-core run with --start-ledger and --start-hash
-func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (runFrom uint32, ledgerHash string, err error) {
+func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (runFrom uint32, err error) {
 	if from == 1 {
 		// Trying to start-from 1 results in an error from Stellar-Core:
 		// Target ledger 1 is not newer than last closed ledger 1 - nothing to do
 		// TODO maybe we can fix it by generating 1st ledger meta
 		// like GenesisLedgerStateReader?
-		err = errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
-		return
+		return 0, errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
 	}
 
 	if from <= 63 {
@@ -298,26 +315,7 @@ func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (ru
 		from = 3
 	}
 
-	runFrom = from - 1
-	if c.ledgerHashStore != nil {
-		var exists bool
-		ledgerHash, exists, err = c.ledgerHashStore.GetLedgerHash(ctx, runFrom)
-		if err != nil {
-			err = errors.Wrapf(err, "error trying to read ledger hash %d", runFrom)
-			return
-		}
-		if exists {
-			return
-		}
-	}
-
-	ledgerHeader, err2 := c.archive.GetLedgerHeader(from)
-	if err2 != nil {
-		err = errors.Wrapf(err2, "error trying to read ledger header %d from HAS", from)
-		return
-	}
-	ledgerHash = hex.EncodeToString(ledgerHeader.Header.PreviousLedgerHash[:])
-	return
+	return from - 1, nil
 }
 
 // nextExpectedSequence returns nextLedger (if currently set) or start of
diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index eda05cf20c..7bbbd6f30e 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -32,8 +32,8 @@ func (m *stellarCoreRunnerMock) catchup(from, to uint32) error {
 	return a.Error(0)
 }
 
-func (m *stellarCoreRunnerMock) runFrom(from uint32, hash string) error {
-	a := m.Called(from, hash)
+func (m *stellarCoreRunnerMock) run() error {
+	a := m.Called()
 	return a.Error(0)
 }
 
@@ -429,9 +429,91 @@ func TestCaptivePrepareRange_ErrCatchup(t *testing.T) {
 	mockRunner.AssertExpectations(t)
 }
 
-func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
+func TestCaptivePrepareRangeUnboundedRange_ErrCatchup(t *testing.T) {
+	mockRunner := &stellarCoreRunnerMock{}
+	mockRunner.On("catchup", uint32(127), uint32(126)).Return(errors.New("transient error")).Once()
+	mockRunner.On("close").Return(nil).Once()
+
+	mockArchive := &historyarchive.MockArchive{}
+	mockArchive.
+		On("GetRootHAS").
+		Return(historyarchive.HistoryArchiveState{
+			CurrentLedger: uint32(127),
+		}, nil)
+
+	ctx := context.Background()
+	cancelCalled := false
+	captiveBackend := CaptiveStellarCore{
+		archive: mockArchive,
+		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
+			return mockRunner, nil
+		},
+		checkpointManager: historyarchive.NewCheckpointManager(64),
+		cancel: context.CancelFunc(func() {
+			cancelCalled = true
+		}),
+	}
+
+	err := captiveBackend.PrepareRange(ctx, UnboundedRange(128))
+	assert.EqualError(t, err, "error starting prepare range: opening subprocess: error running stellar-core: transient error")
+
+	// make sure we can Close without errors
+	assert.NoError(t, captiveBackend.Close())
+	assert.True(t, cancelCalled)
+
+	mockArchive.AssertExpectations(t)
+	mockRunner.AssertExpectations(t)
+}
+
+func TestCaptivePrepareRangeUnboundedRange_CatchupErrExit(t *testing.T) {
+	mockRunner := &stellarCoreRunnerMock{}
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("catchup", uint32(127), uint32(126)).Return(nil).Once()
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(true, errors.New("transient error")).Once()
+	mockRunner.On("close").Return(nil).Once()
+
+	mockArchive := &historyarchive.MockArchive{}
+	mockArchive.
+		On("GetRootHAS").
+		Return(historyarchive.HistoryArchiveState{
+			CurrentLedger: uint32(127),
+		}, nil)
+
+	ctx := context.Background()
+	cancelCalled := false
+	captiveBackend := CaptiveStellarCore{
+		archive: mockArchive,
+		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
+			return mockRunner, nil
+		},
+		checkpointManager: historyarchive.NewCheckpointManager(64),
+		cancel: context.CancelFunc(func() {
+			cancelCalled = true
+		}),
+	}
+
+	err := captiveBackend.PrepareRange(ctx, UnboundedRange(128))
+	assert.EqualError(t, err, "error starting prepare range: opening subprocess: error running stellar-core: transient error")
+
+	// make sure we can Close without errors
+	assert.NoError(t, captiveBackend.Close())
+	assert.True(t, cancelCalled)
+
+	mockArchive.AssertExpectations(t)
+	mockRunner.AssertExpectations(t)
+}
+
+func TestCaptivePrepareRangeUnboundedRange_CatchupHungExit(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(127), "0000000000000000000000000000000000000000000000000000000000000000").Return(errors.New("transient error")).Once()
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("catchup", uint32(127), uint32(126)).Return(nil).Once()
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(false, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
 
 	mockArchive := &historyarchive.MockArchive{}
@@ -441,9 +523,47 @@ func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
 			CurrentLedger: uint32(127),
 		}, nil)
 
+	ctx := context.Background()
+	cancelCalled := false
+	captiveBackend := CaptiveStellarCore{
+		archive: mockArchive,
+		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
+			return mockRunner, nil
+		},
+		checkpointManager: historyarchive.NewCheckpointManager(64),
+		cancel: context.CancelFunc(func() {
+			cancelCalled = true
+		}),
+	}
+
+	err := captiveBackend.PrepareRange(ctx, UnboundedRange(128))
+	assert.EqualError(t, err, "error starting prepare range: opening subprocess: error creating stellar-core runner")
+
+	// make sure we can Close without errors
+	assert.NoError(t, captiveBackend.Close())
+	assert.True(t, cancelCalled)
+
+	mockArchive.AssertExpectations(t)
+	mockRunner.AssertExpectations(t)
+}
+
+func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
+	mockRunner := &stellarCoreRunnerMock{}
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("catchup", uint32(127), uint32(126)).Return(nil).Once()
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("getProcessExitError").Return(true, nil).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("run").Return(errors.New("transient error")).Once()
+	mockRunner.On("close").Return(nil).Once()
+
+	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.
-		On("GetLedgerHeader", uint32(128)).
-		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
+		On("GetRootHAS").
+		Return(historyarchive.HistoryArchiveState{
+			CurrentLedger: uint32(127),
+		}, nil)
 
 	ctx := context.Background()
 	cancelCalled := false
@@ -483,7 +603,13 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(64), "0000000000000000000000000000000000000000000000000000000000000000").Return(nil).Once()
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("catchup", uint32(64), uint32(63)).Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(true, nil).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("run").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -493,9 +619,6 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 		Return(historyarchive.HistoryArchiveState{
 			CurrentLedger: uint32(129),
 		}, nil)
-	mockArchive.
-		On("GetLedgerHeader", uint32(65)).
-		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
 
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
@@ -530,7 +653,13 @@ func TestGetLatestLedgerSequence(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(63), "0000000000000000000000000000000000000000000000000000000000000000").Return(nil).Once()
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("catchup", uint32(63), uint32(62)).Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(true, nil).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("run").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -541,10 +670,6 @@ func TestGetLatestLedgerSequence(t *testing.T) {
 			CurrentLedger: uint32(200),
 		}, nil)
 
-	mockArchive.
-		On("GetLedgerHeader", uint32(64)).
-		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
-
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -668,7 +793,13 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(65), "0101010100000000000000000000000000000000000000000000000000000000").Return(nil).Once()
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("catchup", uint32(65), uint32(64)).Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(true, nil).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("run").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -679,14 +810,6 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) {
 			CurrentLedger: uint32(200),
 		}, nil)
 
-	mockArchive.
-		On("GetLedgerHeader", uint32(66)).
-		Return(xdr.LedgerHeaderHistoryEntry{
-			Header: xdr.LedgerHeader{
-				PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
-			},
-		}, nil).Once()
-
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -776,7 +899,13 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(64), mock.Anything).Return(nil)
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("catchup", uint32(64), uint32(63)).Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(true, nil).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("run").Return(nil)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("close").Return(nil)
@@ -788,10 +917,6 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 			CurrentLedger: uint32(200),
 		}, nil)
 
-	mockArchive.
-		On("GetLedgerHeader", uint32(65)).
-		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
-
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -1163,70 +1288,6 @@ func TestCaptiveGetLedgerTerminatedUnexpectedly(t *testing.T) {
 	}
 }
 
-func TestCaptiveUseOfLedgerHashStore(t *testing.T) {
-	ctx := context.Background()
-	mockArchive := &historyarchive.MockArchive{}
-	mockArchive.
-		On("GetLedgerHeader", uint32(300)).
-		Return(xdr.LedgerHeaderHistoryEntry{
-			Header: xdr.LedgerHeader{
-				PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
-			},
-		}, nil)
-
-	mockLedgerHashStore := &MockLedgerHashStore{}
-	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(1049)).
-		Return("", false, fmt.Errorf("transient error")).Once()
-	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(299)).
-		Return("", false, nil).Once()
-	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(85)).
-		Return("cde", true, nil).Once()
-	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(127)).
-		Return("ghi", true, nil).Once()
-	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(2)).
-		Return("mnb", true, nil).Once()
-
-	cancelCalled := false
-	captiveBackend := CaptiveStellarCore{
-		archive:           mockArchive,
-		ledgerHashStore:   mockLedgerHashStore,
-		checkpointManager: historyarchive.NewCheckpointManager(64),
-		cancel: context.CancelFunc(func() {
-			cancelCalled = true
-		}),
-	}
-
-	runFrom, ledgerHash, err := captiveBackend.runFromParams(ctx, 24)
-	assert.NoError(t, err)
-	assert.Equal(t, uint32(2), runFrom)
-	assert.Equal(t, "mnb", ledgerHash)
-
-	runFrom, ledgerHash, err = captiveBackend.runFromParams(ctx, 86)
-	assert.NoError(t, err)
-	assert.Equal(t, uint32(85), runFrom)
-	assert.Equal(t, "cde", ledgerHash)
-
-	runFrom, ledgerHash, err = captiveBackend.runFromParams(ctx, 128)
-	assert.NoError(t, err)
-	assert.Equal(t, uint32(127), runFrom)
-	assert.Equal(t, "ghi", ledgerHash)
-
-	_, _, err = captiveBackend.runFromParams(ctx, 1050)
-	assert.EqualError(t, err, "error trying to read ledger hash 1049: transient error")
-
-	runFrom, ledgerHash, err = captiveBackend.runFromParams(ctx, 300)
-	assert.NoError(t, err)
-	assert.Equal(t, uint32(299), runFrom, "runFrom")
-	assert.Equal(t, "0101010100000000000000000000000000000000000000000000000000000000", ledgerHash)
-
-	mockLedgerHashStore.On("Close").Return(nil).Once()
-	err = captiveBackend.Close()
-	assert.NoError(t, err)
-	assert.True(t, cancelCalled)
-	mockLedgerHashStore.AssertExpectations(t)
-	mockArchive.AssertExpectations(t)
-}
-
 func TestCaptiveRunFromParams(t *testing.T) {
 	var tests = []struct {
 		from           uint32
@@ -1257,13 +1318,6 @@ func TestCaptiveRunFromParams(t *testing.T) {
 		t.Run(fmt.Sprintf("from_%d", tc.from), func(t *testing.T) {
 			tt := assert.New(t)
 			mockArchive := &historyarchive.MockArchive{}
-			mockArchive.
-				On("GetLedgerHeader", uint32(tc.ledgerArchives)).
-				Return(xdr.LedgerHeaderHistoryEntry{
-					Header: xdr.LedgerHeader{
-						PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
-					},
-				}, nil)
 
 			captiveBackend := CaptiveStellarCore{
 				archive:           mockArchive,
@@ -1271,10 +1325,9 @@ func TestCaptiveRunFromParams(t *testing.T) {
 			}
 
 			ctx := context.Background()
-			runFrom, ledgerHash, err := captiveBackend.runFromParams(ctx, tc.from)
+			runFrom, err := captiveBackend.runFromParams(ctx, tc.from)
 			tt.NoError(err)
 			tt.Equal(tc.runFrom, runFrom, "runFrom")
-			tt.Equal("0101010100000000000000000000000000000000000000000000000000000000", ledgerHash)
 
 			mockArchive.AssertExpectations(t)
 		})
@@ -1373,7 +1426,13 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(299), "0101010100000000000000000000000000000000000000000000000000000000").Return(nil).Once()
+	catchupChan := make(chan metaResult)
+	close(catchupChan)
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
+	mockRunner.On("catchup", uint32(299), uint32(298)).Return(nil).Once()
+	mockRunner.On("getProcessExitError").Return(true, nil).Once()
+	mockRunner.On("close").Return(nil).Once()
+	mockRunner.On("run").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("close").Return(nil).Once()
@@ -1384,17 +1443,8 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 		Return(historyarchive.HistoryArchiveState{
 			CurrentLedger: uint32(255),
 		}, nil)
-	mockArchive.
-		On("GetLedgerHeader", uint32(300)).
-		Return(xdr.LedgerHeaderHistoryEntry{
-			Header: xdr.LedgerHeader{
-				PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
-			},
-		}, nil).Once()
 
 	mockLedgerHashStore := &MockLedgerHashStore{}
-	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(299)).
-		Return("", false, nil).Once()
 
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index 024120001d..f72fa63085 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -23,7 +23,7 @@ import (
 
 type stellarCoreRunnerInterface interface {
 	catchup(from, to uint32) error
-	runFrom(from uint32, hash string) error
+	run() error
 	getMetaPipe() <-chan metaResult
 	context() context.Context
 	getProcessExitError() (bool, error)
@@ -289,8 +289,8 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 	return nil
 }
 
-// runFrom executes the run command with a starting ledger on the captive core subprocess
-func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
+// run executes the run command with a starting ledger on the captive core subprocess
+func (r *stellarCoreRunner) run() error {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
@@ -306,8 +306,6 @@ func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
 	r.cmd = r.createCmd(
 		"run",
 		"--in-memory",
-		"--start-at-ledger", fmt.Sprintf("%d", from),
-		"--start-at-hash", hash,
 		"--metadata-output-stream", r.getPipeName(),
 	)
 

From baa254bd5e04db324df05b001df816d864f6d084 Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Tue, 18 Jan 2022 17:01:24 +0000
Subject: [PATCH 03/17] remove in-memory from core run

---
 ingest/ledgerbackend/stellar_core_runner.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index f72fa63085..54ab9f2066 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -305,7 +305,6 @@ func (r *stellarCoreRunner) run() error {
 
 	r.cmd = r.createCmd(
 		"run",
-		"--in-memory",
 		"--metadata-output-stream", r.getPipeName(),
 	)
 

From 0f274b02a78db83663e6ab418cac7fd2266d1a3e Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Tue, 18 Jan 2022 17:52:33 +0000
Subject: [PATCH 04/17] Remove now-unused ledger hash store

---
 exp/services/captivecore/main.go              | 23 +------
 ingest/ledgerbackend/captive_core_backend.go  | 10 ---
 .../captive_core_backend_test.go              |  4 --
 ingest/ledgerbackend/ledger_hash_store.go     | 62 -------------------
 .../internal/db2/history/ledger_test.go       | 11 +---
 services/horizon/internal/ingest/main.go      |  1 -
 6 files changed, 3 insertions(+), 108 deletions(-)
 delete mode 100644 ingest/ledgerbackend/ledger_hash_store.go

diff --git a/exp/services/captivecore/main.go b/exp/services/captivecore/main.go
index 7fde277d69..dd7f6f2baf 100644
--- a/exp/services/captivecore/main.go
+++ b/exp/services/captivecore/main.go
@@ -13,14 +13,13 @@ import (
 	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/config"
-	"github.com/stellar/go/support/db"
 	supporthttp "github.com/stellar/go/support/http"
 	supportlog "github.com/stellar/go/support/log"
 )
 
 func main() {
 	var port int
-	var networkPassphrase, binaryPath, configPath, dbURL string
+	var networkPassphrase, binaryPath, configPath string
 	var captiveCoreTomlParams ledgerbackend.CaptiveCoreTomlParams
 	var historyArchiveURLs []string
 	var checkpointFrequency uint32
@@ -90,14 +89,6 @@ func main() {
 			},
 			Usage: "minimum log severity (debug, info, warn, error) to log",
 		},
-		&config.ConfigOption{
-			Name:      "db-url",
-			EnvVar:    "DATABASE_URL",
-			ConfigKey: &dbURL,
-			OptType:   types.String,
-			Required:  false,
-			Usage:     "horizon postgres database to connect with",
-		},
 		&config.ConfigOption{
 			Name:           "stellar-captive-core-http-port",
 			ConfigKey:      &captiveCoreTomlParams.HTTPPort,
@@ -141,15 +132,6 @@ func main() {
 				Toml:                captiveCoreToml,
 			}
 
-			var dbConn *db.Session
-			if len(dbURL) > 0 {
-				dbConn, err = db.Open("postgres", dbURL)
-				if err != nil {
-					logger.WithError(err).Fatal("Could not create db connection instance")
-				}
-				captiveConfig.LedgerHashStore = ledgerbackend.NewHorizonDBLedgerHashStore(dbConn)
-			}
-
 			core, err := ledgerbackend.NewCaptive(captiveConfig)
 			if err != nil {
 				logger.WithError(err).Fatal("Could not create captive core instance")
@@ -166,9 +148,6 @@ func main() {
 					// TODO: Check this aborts in-progress requests instead of letting
 					// them finish, to preserve existing behaviour.
 					api.Shutdown()
-					if dbConn != nil {
-						dbConn.Close()
-					}
 				},
 			})
 		},
diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index d4a46b7cd8..4464673c06 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -68,7 +68,6 @@ func (c *CaptiveStellarCore) roundDownToFirstReplayAfterCheckpointStart(ledger u
 type CaptiveStellarCore struct {
 	archive           historyarchive.ArchiveInterface
 	checkpointManager historyarchive.CheckpointManager
-	ledgerHashStore   TrustedLedgerHashStore
 
 	// cancel is the CancelFunc for context which controls the lifetime of a CaptiveStellarCore instance.
 	// Once it is invoked CaptiveStellarCore will not be able to stream ledgers from Stellar Core or
@@ -109,8 +108,6 @@ type CaptiveCoreConfig struct {
 	// CheckpointFrequency is the number of ledgers between checkpoints
 	// if unset, DefaultCheckpointFrequency will be used
 	CheckpointFrequency uint32
-	// LedgerHashStore is an optional store used to obtain hashes for ledger sequences from a trusted source
-	LedgerHashStore TrustedLedgerHashStore
 	// Log is an (optional) custom logger which will capture any output from the Stellar Core process.
 	// If Log is omitted then all output will be printed to stdout.
 	Log *log.Entry
@@ -160,7 +157,6 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 
 	c := &CaptiveStellarCore{
 		archive:           &archivePool,
-		ledgerHashStore:   config.LedgerHashStore,
 		cancel:            cancel,
 		checkpointManager: historyarchive.NewCheckpointManager(config.CheckpointFrequency),
 	}
@@ -641,12 +637,6 @@ func (c *CaptiveStellarCore) Close() error {
 	// after the CaptiveStellarCore context is canceled all subsequent calls to PrepareRange() will fail
 	c.cancel()
 
-	// TODO: Sucks to ignore the error here, but no worse than it was before,
-	// so...
-	if c.ledgerHashStore != nil {
-		c.ledgerHashStore.Close()
-	}
-
 	if c.stellarCoreRunner != nil {
 		return c.stellarCoreRunner.close()
 	}
diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index 7bbbd6f30e..46e4423660 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -1444,14 +1444,11 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 			CurrentLedger: uint32(255),
 		}, nil)
 
-	mockLedgerHashStore := &MockLedgerHashStore{}
-
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
 			return mockRunner, nil
 		},
-		ledgerHashStore:   mockLedgerHashStore,
 		checkpointManager: historyarchive.NewCheckpointManager(64),
 	}
 
@@ -1469,5 +1466,4 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 
 	mockRunner.AssertExpectations(t)
 	mockArchive.AssertExpectations(t)
-	mockLedgerHashStore.AssertExpectations(t)
 }
diff --git a/ingest/ledgerbackend/ledger_hash_store.go b/ingest/ledgerbackend/ledger_hash_store.go
deleted file mode 100644
index be9ebf1bfc..0000000000
--- a/ingest/ledgerbackend/ledger_hash_store.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package ledgerbackend
-
-import (
-	"context"
-
-	sq "github.com/Masterminds/squirrel"
-	"github.com/stretchr/testify/mock"
-
-	"github.com/stellar/go/support/db"
-)
-
-// TrustedLedgerHashStore is used to query ledger data from a trusted source.
-// The store should contain ledgers verified by Stellar-Core, do not use untrusted
-// source like history archives.
-type TrustedLedgerHashStore interface {
-	// GetLedgerHash returns the ledger hash for the given sequence number
-	GetLedgerHash(ctx context.Context, seq uint32) (string, bool, error)
-	Close() error
-}
-
-// HorizonDBLedgerHashStore is a TrustedLedgerHashStore which uses horizon's db to look up ledger hashes
-type HorizonDBLedgerHashStore struct {
-	session db.SessionInterface
-}
-
-// NewHorizonDBLedgerHashStore constructs a new TrustedLedgerHashStore backed by the horizon db
-func NewHorizonDBLedgerHashStore(session db.SessionInterface) TrustedLedgerHashStore {
-	return HorizonDBLedgerHashStore{session: session}
-}
-
-// GetLedgerHash returns the ledger hash for the given sequence number
-func (h HorizonDBLedgerHashStore) GetLedgerHash(ctx context.Context, seq uint32) (string, bool, error) {
-	sql := sq.Select("hl.ledger_hash").From("history_ledgers hl").
-		Limit(1).Where("sequence = ?", seq)
-
-	var hash string
-	err := h.session.Get(ctx, &hash, sql)
-	if h.session.NoRows(err) {
-		return hash, false, nil
-	}
-	return hash, true, err
-}
-
-func (h HorizonDBLedgerHashStore) Close() error {
-	return h.session.Close()
-}
-
-// MockLedgerHashStore is a mock implementation of TrustedLedgerHashStore
-type MockLedgerHashStore struct {
-	mock.Mock
-}
-
-// GetLedgerHash returns the ledger hash for the given sequence number
-func (m *MockLedgerHashStore) GetLedgerHash(ctx context.Context, seq uint32) (string, bool, error) {
-	args := m.Called(ctx, seq)
-	return args.Get(0).(string), args.Get(1).(bool), args.Error(2)
-}
-
-func (m *MockLedgerHashStore) Close() error {
-	args := m.Called()
-	return args.Error(0)
-}
diff --git a/services/horizon/internal/db2/history/ledger_test.go b/services/horizon/internal/db2/history/ledger_test.go
index 0fc5280ed4..f04a1cdd14 100644
--- a/services/horizon/internal/db2/history/ledger_test.go
+++ b/services/horizon/internal/db2/history/ledger_test.go
@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/guregu/null"
-	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/services/horizon/internal/test"
 	"github.com/stellar/go/services/horizon/internal/toid"
 	"github.com/stellar/go/xdr"
@@ -61,10 +60,7 @@ func TestInsertLedger(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	ledgerHashStore := ledgerbackend.NewHorizonDBLedgerHashStore(tt.HorizonSession())
-	_, exists, err := ledgerHashStore.GetLedgerHash(tt.Ctx, 100)
-	tt.Assert.NoError(err)
-	tt.Assert.False(exists)
+	// TODO: Check the ledger doesn't exist in the db
 
 	expectedLedger := Ledger{
 		Sequence:                   69859,
@@ -146,10 +142,7 @@ func TestInsertLedger(t *testing.T) {
 
 	tt.Assert.Equal(expectedLedger, ledgerFromDB)
 
-	hash, exists, err := ledgerHashStore.GetLedgerHash(tt.Ctx, uint32(expectedLedger.Sequence))
-	tt.Assert.NoError(err)
-	tt.Assert.True(exists)
-	tt.Assert.Equal(expectedLedger.LedgerHash, hash)
+	// TODO: Check the ledger does exist in the db
 }
 
 func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) {
diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go
index 53abbfa9e8..be30c1de4c 100644
--- a/services/horizon/internal/ingest/main.go
+++ b/services/horizon/internal/ingest/main.go
@@ -228,7 +228,6 @@ func NewSystem(config Config) (System, error) {
 					NetworkPassphrase:   config.NetworkPassphrase,
 					HistoryArchiveURLs:  []string{config.HistoryArchiveURL},
 					CheckpointFrequency: config.CheckpointFrequency,
-					LedgerHashStore:     ledgerbackend.NewHorizonDBLedgerHashStore(config.HistorySession),
 					Log:                 logger,
 					Context:             ctx,
 				},

From 031cda34d990d64d9570deb512097060d64e4aca Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Tue, 18 Jan 2022 18:03:27 +0000
Subject: [PATCH 05/17] fix govet

---
 ingest/ledgerbackend/captive_core_backend.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 4464673c06..78ad8e8284 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -260,11 +260,12 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	for range c.stellarCoreRunner.getMetaPipe() {
 		// Drain the pipe
 	}
-	if err := c.stellarCoreRunner.close(); err != nil {
+	if err = c.stellarCoreRunner.close(); err != nil {
 		return errors.Wrap(err, "error running stellar-core")
 	}
 
-	if processExited, err := c.stellarCoreRunner.getProcessExitError(); err != nil {
+	var processExited bool
+	if processExited, err = c.stellarCoreRunner.getProcessExitError(); err != nil {
 		return errors.Wrap(err, "error running stellar-core")
 	} else if !processExited {
 		return errors.New("error creating stellar-core runner")

From a3496b51a794cab463540374256449678b1074cb Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Wed, 19 Jan 2022 16:46:00 +0000
Subject: [PATCH 06/17] Can't catchup to ledger 1

---
 ingest/ledgerbackend/captive_core_backend.go  | 38 ++++++++++---------
 .../captive_core_backend_test.go              | 18 ++++-----
 2 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 78ad8e8284..cf82e40008 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -253,22 +253,25 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 		return errors.Wrap(err, "error calculating ledger and hash for stellar-core run")
 	}
 
-	err = c.stellarCoreRunner.catchup(runFrom, runFrom-1)
-	if err != nil {
-		return errors.Wrap(err, "error running stellar-core")
-	}
-	for range c.stellarCoreRunner.getMetaPipe() {
-		// Drain the pipe
-	}
-	if err = c.stellarCoreRunner.close(); err != nil {
-		return errors.Wrap(err, "error running stellar-core")
-	}
+	if runFrom > 1 {
+		// Can't catch up to the first ledger
+		err = c.stellarCoreRunner.catchup(runFrom-1, runFrom)
+		if err != nil {
+			return errors.Wrap(err, "error running stellar-core")
+		}
+		for range c.stellarCoreRunner.getMetaPipe() {
+			// Drain the pipe
+		}
+		if err = c.stellarCoreRunner.close(); err != nil {
+			return errors.Wrap(err, "error running stellar-core")
+		}
 
-	var processExited bool
-	if processExited, err = c.stellarCoreRunner.getProcessExitError(); err != nil {
-		return errors.Wrap(err, "error running stellar-core")
-	} else if !processExited {
-		return errors.New("error creating stellar-core runner")
+		var processExited bool
+		if processExited, err = c.stellarCoreRunner.getProcessExitError(); err != nil {
+			return errors.Wrap(err, "error running stellar-core")
+		} else if !processExited {
+			return errors.New("error creating stellar-core runner")
+		}
 	}
 
 	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline); err != nil {
@@ -293,7 +296,7 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	return nil
 }
 
-// runFromParams receives a ledger sequence and calculates the required values to call stellar-core run with --start-ledger and --start-hash
+// runFromParams receives a ledger sequence and calculates the required values to start stellar-core catchup from
 func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (runFrom uint32, err error) {
 	if from == 1 {
 		// Trying to start-from 1 results in an error from Stellar-Core:
@@ -312,7 +315,8 @@ func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (ru
 		from = 3
 	}
 
-	return from - 1, nil
+	runFrom = from - 1
+	return
 }
 
 // nextExpectedSequence returns nextLedger (if currently set) or start of
diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index 46e4423660..90d60b42fe 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -431,7 +431,7 @@ func TestCaptivePrepareRange_ErrCatchup(t *testing.T) {
 
 func TestCaptivePrepareRangeUnboundedRange_ErrCatchup(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("catchup", uint32(127), uint32(126)).Return(errors.New("transient error")).Once()
+	mockRunner.On("catchup", uint32(126), uint32(127)).Return(errors.New("transient error")).Once()
 	mockRunner.On("close").Return(nil).Once()
 
 	mockArchive := &historyarchive.MockArchive{}
@@ -469,7 +469,7 @@ func TestCaptivePrepareRangeUnboundedRange_CatchupErrExit(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
-	mockRunner.On("catchup", uint32(127), uint32(126)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(126), uint32(127)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(true, errors.New("transient error")).Once()
@@ -510,7 +510,7 @@ func TestCaptivePrepareRangeUnboundedRange_CatchupHungExit(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
-	mockRunner.On("catchup", uint32(127), uint32(126)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(126), uint32(127)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(false, nil).Once()
@@ -551,7 +551,7 @@ func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
-	mockRunner.On("catchup", uint32(127), uint32(126)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(126), uint32(127)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
 	mockRunner.On("getProcessExitError").Return(true, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
@@ -606,7 +606,7 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(64), uint32(63)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(63), uint32(64)).Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(true, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("run").Return(nil).Once()
@@ -656,7 +656,7 @@ func TestGetLatestLedgerSequence(t *testing.T) {
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(63), uint32(62)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(62), uint32(63)).Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(true, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("run").Return(nil).Once()
@@ -796,7 +796,7 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) {
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(65), uint32(64)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(64), uint32(65)).Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(true, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("run").Return(nil).Once()
@@ -902,7 +902,7 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(64), uint32(63)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(63), uint32(64)).Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(true, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("run").Return(nil)
@@ -1429,7 +1429,7 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 	catchupChan := make(chan metaResult)
 	close(catchupChan)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(299), uint32(298)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(298), uint32(299)).Return(nil).Once()
 	mockRunner.On("getProcessExitError").Return(true, nil).Once()
 	mockRunner.On("close").Return(nil).Once()
 	mockRunner.On("run").Return(nil).Once()

From f4687999b8c52a8f54bf665e4bfcd04fd968bcb7 Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Tue, 25 Jan 2022 19:16:38 +0000
Subject: [PATCH 07/17] catchup from x-1, not from x

---
 ingest/ledgerbackend/captive_core_backend.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index cf82e40008..92743b5bb4 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -253,9 +253,9 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 		return errors.Wrap(err, "error calculating ledger and hash for stellar-core run")
 	}
 
-	if runFrom > 1 {
+	if runFrom > 2 {
 		// Can't catch up to the first ledger
-		err = c.stellarCoreRunner.catchup(runFrom-1, runFrom)
+		err = c.stellarCoreRunner.catchup(runFrom-2, runFrom-1)
 		if err != nil {
 			return errors.Wrap(err, "error running stellar-core")
 		}

From f1789e8eb7f28ee1be88d0f2034f55d73e255d6d Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Wed, 26 Jan 2022 14:53:01 +0000
Subject: [PATCH 08/17] Add database= flag to the captive core config file

---
 ingest/ledgerbackend/testdata/expected-offline-core.cfg  | 1 +
 .../testdata/expected-offline-with-appendix-core.cfg     | 1 +
 .../testdata/expected-offline-with-extra-fields.cfg      | 1 +
 .../testdata/expected-offline-with-no-peer-port.cfg      | 1 +
 ingest/ledgerbackend/testdata/expected-online-core.cfg   | 1 +
 .../testdata/expected-online-with-no-http-port.cfg       | 1 +
 .../testdata/expected-online-with-no-peer-port.cfg       | 1 +
 ingest/ledgerbackend/toml.go                             | 9 +++++++++
 8 files changed, 16 insertions(+)

diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
index 62aeeb6664..d6a80a628d 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
index 124abc435b..178764beab 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
index d7b41a0b81..a0f54c6f70 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
index 9eca1ccad1..e5acfe73e6 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg
index 57a5e7ff2c..1fa85f8b84 100644
--- a/ingest/ledgerbackend/testdata/expected-online-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
index 89e1762757..d164a8d67b 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
index 1b65c5f318..4ff5d3477a 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
index c6ac869a9f..292a45b72d 100644
--- a/ingest/ledgerbackend/toml.go
+++ b/ingest/ledgerbackend/toml.go
@@ -15,6 +15,7 @@ import (
 )
 
 const (
+	defaultDatabase      = "sqlite3://stellar.db"
 	defaultHTTPPort      = 11626
 	defaultFailureSafety = -1
 
@@ -61,6 +62,7 @@ type QuorumSet struct {
 }
 
 type captiveCoreTomlValues struct {
+	Database string `toml:"DATABASE,omitempty"`
 	// we cannot omitempty because the empty string is a valid configuration for LOG_FILE_PATH
 	// and the default is stellar-core.log
 	LogFilePath   string `toml:"LOG_FILE_PATH"`
@@ -405,6 +407,11 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) {
 }
 
 func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
+	// TODO: Do we need to skip this if we are in --in-memory mode?
+	if !c.tree.Has("DATABASE") {
+		c.Database = defaultDatabase
+	}
+
 	if !c.tree.Has("NETWORK_PASSPHRASE") {
 		c.NetworkPassphrase = params.NetworkPassphrase
 	}
@@ -440,6 +447,8 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 }
 
 func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error {
+	// TODO: Any way to validate database here?
+
 	if def := c.tree.Has("NETWORK_PASSPHRASE"); def && c.NetworkPassphrase != params.NetworkPassphrase {
 		return fmt.Errorf(
 			"NETWORK_PASSPHRASE in captive core config file: %s does not match Horizon network-passphrase flag: %s",

From 760ba2fcfb556692fed3555e71320f84695e0408 Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Wed, 26 Jan 2022 15:09:33 +0000
Subject: [PATCH 09/17] Have to run stellar-core new-db to initialize the new
 db

---
 ingest/ledgerbackend/stellar_core_runner.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index 54ab9f2066..31aafcb6c9 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -260,6 +260,10 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 		return errors.New("runner already started")
 	}
 
+	if err := r.createCmd("new-db").Run(); err != nil {
+		return errors.Wrap(err, "error initializing core db")
+	}
+
 	rangeArg := fmt.Sprintf("%d/%d", to, to-from+1)
 	r.cmd = r.createCmd(
 		"catchup", rangeArg,
@@ -303,6 +307,10 @@ func (r *stellarCoreRunner) run() error {
 		return errors.New("runner already started")
 	}
 
+	if err := r.createCmd("new-db").Run(); err != nil {
+		return errors.Wrap(err, "error initializing core db")
+	}
+
 	r.cmd = r.createCmd(
 		"run",
 		"--metadata-output-stream", r.getPipeName(),

From 2053b978949faea77d57b8f2f0eea3f2f38b2a41 Mon Sep 17 00:00:00 2001
From: Paul Bellamy <paul@stellar.org>
Date: Fri, 28 Jan 2022 10:07:17 +0000
Subject: [PATCH 10/17] Experiment with pushing the catchup into
 stellar_core_runner to use the same directory

---
 ingest/ledgerbackend/captive_core_backend.go  |  35 +---
 .../captive_core_backend_test.go              | 170 +-----------------
 ingest/ledgerbackend/stellar_core_runner.go   |  19 +-
 3 files changed, 30 insertions(+), 194 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 92743b5bb4..ecf61fad01 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -243,43 +243,20 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	}
 
 	var runner stellarCoreRunnerInterface
-	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOffline); err != nil {
+	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline); err != nil {
 		return errors.Wrap(err, "error creating stellar-core runner")
+	} else {
+		// only assign c.stellarCoreRunner if runner is not nil to avoid nil interface check
+		// see https://golang.org/doc/faq#nil_error
+		c.stellarCoreRunner = runner
 	}
-	c.stellarCoreRunner = runner
 
 	runFrom, err := c.runFromParams(ctx, from)
 	if err != nil {
 		return errors.Wrap(err, "error calculating ledger and hash for stellar-core run")
 	}
 
-	if runFrom > 2 {
-		// Can't catch up to the first ledger
-		err = c.stellarCoreRunner.catchup(runFrom-2, runFrom-1)
-		if err != nil {
-			return errors.Wrap(err, "error running stellar-core")
-		}
-		for range c.stellarCoreRunner.getMetaPipe() {
-			// Drain the pipe
-		}
-		if err = c.stellarCoreRunner.close(); err != nil {
-			return errors.Wrap(err, "error running stellar-core")
-		}
-
-		var processExited bool
-		if processExited, err = c.stellarCoreRunner.getProcessExitError(); err != nil {
-			return errors.Wrap(err, "error running stellar-core")
-		} else if !processExited {
-			return errors.New("error creating stellar-core runner")
-		}
-	}
-
-	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline); err != nil {
-		return errors.Wrap(err, "error creating stellar-core runner")
-	}
-	c.stellarCoreRunner = runner
-
-	err = c.stellarCoreRunner.run()
+	err = c.stellarCoreRunner.runFrom(runFrom)
 	if err != nil {
 		return errors.Wrap(err, "error running stellar-core")
 	}
diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index 90d60b42fe..6eb0e3da87 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -32,8 +32,8 @@ func (m *stellarCoreRunnerMock) catchup(from, to uint32) error {
 	return a.Error(0)
 }
 
-func (m *stellarCoreRunnerMock) run() error {
-	a := m.Called()
+func (m *stellarCoreRunnerMock) runFrom(from uint32) error {
+	a := m.Called(from)
 	return a.Error(0)
 }
 
@@ -429,133 +429,9 @@ func TestCaptivePrepareRange_ErrCatchup(t *testing.T) {
 	mockRunner.AssertExpectations(t)
 }
 
-func TestCaptivePrepareRangeUnboundedRange_ErrCatchup(t *testing.T) {
-	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("catchup", uint32(126), uint32(127)).Return(errors.New("transient error")).Once()
-	mockRunner.On("close").Return(nil).Once()
-
-	mockArchive := &historyarchive.MockArchive{}
-	mockArchive.
-		On("GetRootHAS").
-		Return(historyarchive.HistoryArchiveState{
-			CurrentLedger: uint32(127),
-		}, nil)
-
-	ctx := context.Background()
-	cancelCalled := false
-	captiveBackend := CaptiveStellarCore{
-		archive: mockArchive,
-		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
-			return mockRunner, nil
-		},
-		checkpointManager: historyarchive.NewCheckpointManager(64),
-		cancel: context.CancelFunc(func() {
-			cancelCalled = true
-		}),
-	}
-
-	err := captiveBackend.PrepareRange(ctx, UnboundedRange(128))
-	assert.EqualError(t, err, "error starting prepare range: opening subprocess: error running stellar-core: transient error")
-
-	// make sure we can Close without errors
-	assert.NoError(t, captiveBackend.Close())
-	assert.True(t, cancelCalled)
-
-	mockArchive.AssertExpectations(t)
-	mockRunner.AssertExpectations(t)
-}
-
-func TestCaptivePrepareRangeUnboundedRange_CatchupErrExit(t *testing.T) {
-	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("catchup", uint32(126), uint32(127)).Return(nil).Once()
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(true, errors.New("transient error")).Once()
-	mockRunner.On("close").Return(nil).Once()
-
-	mockArchive := &historyarchive.MockArchive{}
-	mockArchive.
-		On("GetRootHAS").
-		Return(historyarchive.HistoryArchiveState{
-			CurrentLedger: uint32(127),
-		}, nil)
-
-	ctx := context.Background()
-	cancelCalled := false
-	captiveBackend := CaptiveStellarCore{
-		archive: mockArchive,
-		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
-			return mockRunner, nil
-		},
-		checkpointManager: historyarchive.NewCheckpointManager(64),
-		cancel: context.CancelFunc(func() {
-			cancelCalled = true
-		}),
-	}
-
-	err := captiveBackend.PrepareRange(ctx, UnboundedRange(128))
-	assert.EqualError(t, err, "error starting prepare range: opening subprocess: error running stellar-core: transient error")
-
-	// make sure we can Close without errors
-	assert.NoError(t, captiveBackend.Close())
-	assert.True(t, cancelCalled)
-
-	mockArchive.AssertExpectations(t)
-	mockRunner.AssertExpectations(t)
-}
-
-func TestCaptivePrepareRangeUnboundedRange_CatchupHungExit(t *testing.T) {
-	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("catchup", uint32(126), uint32(127)).Return(nil).Once()
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(false, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-
-	mockArchive := &historyarchive.MockArchive{}
-	mockArchive.
-		On("GetRootHAS").
-		Return(historyarchive.HistoryArchiveState{
-			CurrentLedger: uint32(127),
-		}, nil)
-
-	ctx := context.Background()
-	cancelCalled := false
-	captiveBackend := CaptiveStellarCore{
-		archive: mockArchive,
-		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
-			return mockRunner, nil
-		},
-		checkpointManager: historyarchive.NewCheckpointManager(64),
-		cancel: context.CancelFunc(func() {
-			cancelCalled = true
-		}),
-	}
-
-	err := captiveBackend.PrepareRange(ctx, UnboundedRange(128))
-	assert.EqualError(t, err, "error starting prepare range: opening subprocess: error creating stellar-core runner")
-
-	// make sure we can Close without errors
-	assert.NoError(t, captiveBackend.Close())
-	assert.True(t, cancelCalled)
-
-	mockArchive.AssertExpectations(t)
-	mockRunner.AssertExpectations(t)
-}
-
 func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("catchup", uint32(126), uint32(127)).Return(nil).Once()
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("getProcessExitError").Return(true, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("run").Return(errors.New("transient error")).Once()
+	mockRunner.On("runFrom", uint32(127)).Return(errors.New("transient error")).Once()
 	mockRunner.On("close").Return(nil).Once()
 
 	mockArchive := &historyarchive.MockArchive{}
@@ -603,13 +479,7 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(63), uint32(64)).Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(true, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("run").Return(nil).Once()
+	mockRunner.On("runFrom", uint32(64)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -653,13 +523,7 @@ func TestGetLatestLedgerSequence(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(62), uint32(63)).Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(true, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("run").Return(nil).Once()
+	mockRunner.On("runFrom", uint32(63)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -793,13 +657,7 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(64), uint32(65)).Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(true, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("run").Return(nil).Once()
+	mockRunner.On("runFrom", uint32(65)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -899,13 +757,7 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(63), uint32(64)).Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(true, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("run").Return(nil)
+	mockRunner.On("runFrom", uint32(64)).Return(nil)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("close").Return(nil)
@@ -1426,13 +1278,7 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	catchupChan := make(chan metaResult)
-	close(catchupChan)
-	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(catchupChan)).Once()
-	mockRunner.On("catchup", uint32(298), uint32(299)).Return(nil).Once()
-	mockRunner.On("getProcessExitError").Return(true, nil).Once()
-	mockRunner.On("close").Return(nil).Once()
-	mockRunner.On("run").Return(nil).Once()
+	mockRunner.On("runFrom", uint32(299)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("close").Return(nil).Once()
diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index 31aafcb6c9..9d87956be2 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -23,7 +23,7 @@ import (
 
 type stellarCoreRunnerInterface interface {
 	catchup(from, to uint32) error
-	run() error
+	runFrom(from uint32) error
 	getMetaPipe() <-chan metaResult
 	context() context.Context
 	getProcessExitError() (bool, error)
@@ -293,8 +293,8 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 	return nil
 }
 
-// run executes the run command with a starting ledger on the captive core subprocess
-func (r *stellarCoreRunner) run() error {
+// runFrom executes the run command with a starting ledger on the captive core subprocess
+func (r *stellarCoreRunner) runFrom(from uint32) error {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
@@ -311,6 +311,19 @@ func (r *stellarCoreRunner) run() error {
 		return errors.Wrap(err, "error initializing core db")
 	}
 
+	// Do a quick catch-up to set the LCL in core to be our expected starting
+	// point.
+	//
+	// TODO: If we're re-using the directory from a previous run, we need to do
+	// this differently, because the LCL in core might already be ahead of our
+	// `from` point.
+	if from > 2 {
+		// Can't catch up to the genesis ledger.
+		if err := r.createCmd("catchup", fmt.Sprintf("%d/0", from-1)).Run(); err != nil {
+			return errors.Wrap(err, "error runing stellar-core catchup")
+		}
+	}
+
 	r.cmd = r.createCmd(
 		"run",
 		"--metadata-output-stream", r.getPipeName(),

From 094468aa7a7f5e1a77db2734000b3d145e93c6ff Mon Sep 17 00:00:00 2001
From: shawn <sreuland@users.noreply.github.com>
Date: Mon, 31 Jan 2022 10:59:51 -0800
Subject: [PATCH 11/17] #4038: fixed unit test in ingest/ledgerbackend to mock
 for new captive core method usages

---
 ingest/ledgerbackend/captive_core_backend.go        | 11 ++++++-----
 ingest/ledgerbackend/captive_core_backend_test.go   |  8 +++++++-
 .../internal/test/integration/integration.go        | 13 +++++++++++--
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index ecf61fad01..74be8fa426 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -138,6 +138,7 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 	if parentCtx == nil {
 		parentCtx = context.Background()
 	}
+
 	var cancel context.CancelFunc
 	config.Context, cancel = context.WithCancel(parentCtx)
 
@@ -245,11 +246,8 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	var runner stellarCoreRunnerInterface
 	if runner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline); err != nil {
 		return errors.Wrap(err, "error creating stellar-core runner")
-	} else {
-		// only assign c.stellarCoreRunner if runner is not nil to avoid nil interface check
-		// see https://golang.org/doc/faq#nil_error
-		c.stellarCoreRunner = runner
 	}
+	c.stellarCoreRunner = runner
 
 	runFrom, err := c.runFromParams(ctx, from)
 	if err != nil {
@@ -381,9 +379,12 @@ func (c *CaptiveStellarCore) isPrepared(ledgerRange Range) bool {
 	if c.stellarCoreRunner == nil {
 		return false
 	}
-	if c.closed {
+
+	exited, _ := c.stellarCoreRunner.getProcessExitError()
+	if exited {
 		return false
 	}
+
 	lastLedger := uint32(0)
 	if c.lastLedger != nil {
 		lastLedger = *c.lastLedger
diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index 6eb0e3da87..47fd3abe3d 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -277,9 +277,11 @@ func TestCaptivePrepareRangeCloseNotFullyTerminated(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("catchup", uint32(100), uint32(200)).Return(nil).Once()
+	mockRunner.On("catchup", uint32(100), uint32(200)).Return(nil).Twice()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
+	mockRunner.On("getProcessExitError").Return(true, nil)
+	mockRunner.On("close").Return(nil)
 
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.
@@ -313,6 +315,7 @@ func TestCaptivePrepareRange_ErrClosingSession(t *testing.T) {
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
 	mockRunner.On("close").Return(fmt.Errorf("transient error"))
+	mockRunner.On("getProcessExitError").Return(false, nil)
 
 	captiveBackend := CaptiveStellarCore{
 		nextLedger:        300,
@@ -482,6 +485,7 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 	mockRunner.On("runFrom", uint32(64)).Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
+	mockRunner.On("getProcessExitError").Return(false, nil)
 
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.
@@ -570,6 +574,7 @@ func TestCaptiveGetLedger(t *testing.T) {
 	mockRunner.On("catchup", uint32(65), uint32(66)).Return(nil)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
+	mockRunner.On("getProcessExitError").Return(false, nil)
 
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.
@@ -1189,6 +1194,7 @@ func TestCaptiveRunFromParams(t *testing.T) {
 func TestCaptiveIsPrepared(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
 	mockRunner.On("context").Return(context.Background()).Maybe()
+	mockRunner.On("getProcessExitError").Return(false, nil)
 
 	// c.prepared == nil
 	captiveBackend := CaptiveStellarCore{
diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go
index e5a265da97..c880fa5841 100644
--- a/services/horizon/internal/test/integration/integration.go
+++ b/services/horizon/internal/test/integration/integration.go
@@ -358,8 +358,17 @@ func (i *Test) StartHorizon() error {
 		HorizonURL: fmt.Sprintf("http://%s:%s", hostname, horizonPort),
 	}
 
-	if err = i.app.Ingestion().BuildGenesisState(); err != nil {
-		return errors.Wrap(err, "cannot build genesis state")
+	if !RunWithCaptiveCore {
+		// captive core backend ledger is executed with sqlite on db, not memory, which results in
+		// different meta stream from 'steallar-core run' , it does not replay meta close stream from genesis during 'run' after 'new-db'
+		// rather at start of 'run' core will fast forward internally the ClosedMeta stream to start emitting ledgers from
+		// its current LCL(last committed ledger, not necessarily the latest from network side).
+		// Consequently, Don't want tests that are running captive core to initially set horizon ingest state to genesis here,
+		// the fsm would then get stuck in state loop since core is only emitting meta close ledger seqs that are > than genesis
+		// sequence of '2' which horizon's fsm keeps requesting as next ledger.
+		if err = i.app.Ingestion().BuildGenesisState(); err != nil {
+			return errors.Wrap(err, "cannot build genesis state")
+		}
 	}
 
 	done := make(chan struct{})

From ac70788c5d14b204812788264eff0fd2e44f1117 Mon Sep 17 00:00:00 2001
From: Shawn Reuland <shawn@stellar.org>
Date: Mon, 31 Jan 2022 15:01:57 -0800
Subject: [PATCH 12/17] #4038: fixed unit test after merge conflict

---
 ingest/ledgerbackend/captive_core_backend_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index 11fb9f77cb..a213551c02 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -1260,6 +1260,7 @@ func TestCaptiveIsPreparedCoreContextCancelled(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
 	ctx, cancel := context.WithCancel(context.Background())
 	mockRunner.On("context").Return(ctx).Maybe()
+	mockRunner.On("getProcessExitError").Return(false, nil)
 
 	rang := UnboundedRange(100)
 	captiveBackend := CaptiveStellarCore{

From f16f43cef1f0616b1819979b46a24da8aff4f2c9 Mon Sep 17 00:00:00 2001
From: Shawn Reuland <shawn@stellar.org>
Date: Tue, 1 Feb 2022 14:39:13 -0800
Subject: [PATCH 13/17] #4038: fixed integration test to not use GenesisState

---
 ingest/ledgerbackend/stellar_core_runner.go         |  7 ++++---
 ingest/ledgerbackend/toml.go                        |  3 ---
 services/horizon/internal/app.go                    |  5 -----
 .../internal/test/integration/integration.go        | 13 -------------
 4 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index 9d87956be2..4ebd791f5e 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -314,14 +314,15 @@ func (r *stellarCoreRunner) runFrom(from uint32) error {
 	// Do a quick catch-up to set the LCL in core to be our expected starting
 	// point.
 	//
-	// TODO: If we're re-using the directory from a previous run, we need to do
-	// this differently, because the LCL in core might already be ahead of our
-	// `from` point.
 	if from > 2 {
 		// Can't catch up to the genesis ledger.
 		if err := r.createCmd("catchup", fmt.Sprintf("%d/0", from-1)).Run(); err != nil {
 			return errors.Wrap(err, "error runing stellar-core catchup")
 		}
+	} else {
+		if err := r.createCmd("catchup", "2/0").Run(); err != nil {
+			return errors.Wrap(err, "error runing stellar-core catchup")
+		}
 	}
 
 	r.cmd = r.createCmd(
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
index 292a45b72d..adc0dcf577 100644
--- a/ingest/ledgerbackend/toml.go
+++ b/ingest/ledgerbackend/toml.go
@@ -407,7 +407,6 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) {
 }
 
 func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
-	// TODO: Do we need to skip this if we are in --in-memory mode?
 	if !c.tree.Has("DATABASE") {
 		c.Database = defaultDatabase
 	}
@@ -447,8 +446,6 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 }
 
 func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error {
-	// TODO: Any way to validate database here?
-
 	if def := c.tree.Has("NETWORK_PASSPHRASE"); def && c.NetworkPassphrase != params.NetworkPassphrase {
 		return fmt.Errorf(
 			"NETWORK_PASSPHRASE in captive core config file: %s does not match Horizon network-passphrase flag: %s",
diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go
index f3fa68701f..178595c411 100644
--- a/services/horizon/internal/app.go
+++ b/services/horizon/internal/app.go
@@ -195,11 +195,6 @@ func (a *App) HistoryQ() *history.Q {
 	return a.historyQ
 }
 
-// Ingestion returns the ingestion system associated with this Horizon instance
-func (a *App) Ingestion() ingest.System {
-	return a.ingester
-}
-
 // HorizonSession returns a new session that loads data from the horizon
 // database.
 func (a *App) HorizonSession() db.SessionInterface {
diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go
index c880fa5841..51cb2eb1de 100644
--- a/services/horizon/internal/test/integration/integration.go
+++ b/services/horizon/internal/test/integration/integration.go
@@ -358,19 +358,6 @@ func (i *Test) StartHorizon() error {
 		HorizonURL: fmt.Sprintf("http://%s:%s", hostname, horizonPort),
 	}
 
-	if !RunWithCaptiveCore {
-		// captive core backend ledger is executed with sqlite on db, not memory, which results in
-		// different meta stream from 'steallar-core run' , it does not replay meta close stream from genesis during 'run' after 'new-db'
-		// rather at start of 'run' core will fast forward internally the ClosedMeta stream to start emitting ledgers from
-		// its current LCL(last committed ledger, not necessarily the latest from network side).
-		// Consequently, Don't want tests that are running captive core to initially set horizon ingest state to genesis here,
-		// the fsm would then get stuck in state loop since core is only emitting meta close ledger seqs that are > than genesis
-		// sequence of '2' which horizon's fsm keeps requesting as next ledger.
-		if err = i.app.Ingestion().BuildGenesisState(); err != nil {
-			return errors.Wrap(err, "cannot build genesis state")
-		}
-	}
-
 	done := make(chan struct{})
 	go func() {
 		i.app.Serve()

From a0ce7451c73b2baa327b2a6b0619b150563d633d Mon Sep 17 00:00:00 2001
From: shawn <sreuland@users.noreply.github.com>
Date: Wed, 16 Feb 2022 10:56:12 -0800
Subject: [PATCH 14/17] #4038: use --in-memory for cc ingest by default

only use on disk with new captive-core-use-external-storage=true cli param
---
 .circleci/config.yml                          |  13 ++
 exp/services/captivecore/main.go              |  23 +++-
 ingest/ledgerbackend/captive_core_backend.go  |  46 ++++++-
 .../captive_core_backend_test.go              | 126 ++++++++++++++++--
 ingest/ledgerbackend/ledger_hash_store.go     |  62 +++++++++
 ingest/ledgerbackend/stellar_core_runner.go   | 100 +++++++++-----
 .../testdata/expected-offline-core.cfg        |   1 -
 .../expected-offline-with-appendix-core.cfg   |   1 -
 .../expected-offline-with-extra-fields.cfg    |   1 -
 .../expected-offline-with-no-peer-port.cfg    |   1 -
 .../testdata/expected-online-core.cfg         |   1 -
 .../expected-online-with-no-http-port.cfg     |   1 -
 .../expected-online-with-no-peer-port.cfg     |   1 -
 ingest/ledgerbackend/toml.go                  |  19 ++-
 ingest/ledgerbackend/toml_test.go             |  88 +++++++++++-
 integration.sh                                |   1 +
 services/horizon/CHANGELOG.md                 |   2 +
 services/horizon/cmd/db.go                    |  27 ++--
 services/horizon/cmd/ingest.go                |  21 +--
 services/horizon/internal/config.go           |  19 +--
 services/horizon/internal/flags.go            |  14 ++
 services/horizon/internal/ingest/main.go      |  22 +--
 services/horizon/internal/init.go             |  23 ++--
 .../horizon/internal/integration/db_test.go   |   2 +
 .../internal/test/integration/integration.go  |  35 +++--
 25 files changed, 521 insertions(+), 129 deletions(-)
 create mode 100644 ingest/ledgerbackend/ledger_hash_store.go

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 50f99917e1..a0c175c7fb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -430,6 +430,9 @@ jobs:
       enable-captive-core:
         type: boolean
         default: false
+      enable-captive-core-remote-storage:
+        type: boolean
+        default: false  
     working_directory: ~/go/src/github.com/stellar/go
     machine:
       image: ubuntu-2004:202010-01
@@ -452,6 +455,12 @@ jobs:
             - run:
                 name: Setting Captive Core env variables
                 command: echo "export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true" >> $BASH_ENV
+      - when:
+          condition: <<parameters.enable-captive-core-remote-storage>>
+          steps:
+            - run:
+                name: Setting Captive Core Remote Storage env variable
+                command: echo "export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_REMOTE_STORAGE=true" >> $BASH_ENV          
       - run:
           name: Run Horizon integration tests <<#parameters.enable-captive-core>>(With captive core)<</parameters.enable-captive-core>>
           # Currently all integration tests are in a single directory.
@@ -480,6 +489,10 @@ workflows:
       - test_horizon_integration:
           name: test_horizon_integration_with_captive_core
           enable-captive-core: true
+      - test_horizon_integration:
+          name: test_horizon_integration_with_captive_core_remote_storage
+          enable-captive-core: true   
+          enable-captive-core-remote-storage: true  
       - test_verify_range_docker_image:
           filters:
               # we use test_verify_range_docker_image with publish in master
diff --git a/exp/services/captivecore/main.go b/exp/services/captivecore/main.go
index dd7f6f2baf..7fde277d69 100644
--- a/exp/services/captivecore/main.go
+++ b/exp/services/captivecore/main.go
@@ -13,13 +13,14 @@ import (
 	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/config"
+	"github.com/stellar/go/support/db"
 	supporthttp "github.com/stellar/go/support/http"
 	supportlog "github.com/stellar/go/support/log"
 )
 
 func main() {
 	var port int
-	var networkPassphrase, binaryPath, configPath string
+	var networkPassphrase, binaryPath, configPath, dbURL string
 	var captiveCoreTomlParams ledgerbackend.CaptiveCoreTomlParams
 	var historyArchiveURLs []string
 	var checkpointFrequency uint32
@@ -89,6 +90,14 @@ func main() {
 			},
 			Usage: "minimum log severity (debug, info, warn, error) to log",
 		},
+		&config.ConfigOption{
+			Name:      "db-url",
+			EnvVar:    "DATABASE_URL",
+			ConfigKey: &dbURL,
+			OptType:   types.String,
+			Required:  false,
+			Usage:     "horizon postgres database to connect with",
+		},
 		&config.ConfigOption{
 			Name:           "stellar-captive-core-http-port",
 			ConfigKey:      &captiveCoreTomlParams.HTTPPort,
@@ -132,6 +141,15 @@ func main() {
 				Toml:                captiveCoreToml,
 			}
 
+			var dbConn *db.Session
+			if len(dbURL) > 0 {
+				dbConn, err = db.Open("postgres", dbURL)
+				if err != nil {
+					logger.WithError(err).Fatal("Could not create db connection instance")
+				}
+				captiveConfig.LedgerHashStore = ledgerbackend.NewHorizonDBLedgerHashStore(dbConn)
+			}
+
 			core, err := ledgerbackend.NewCaptive(captiveConfig)
 			if err != nil {
 				logger.WithError(err).Fatal("Could not create captive core instance")
@@ -148,6 +166,9 @@ func main() {
 					// TODO: Check this aborts in-progress requests instead of letting
 					// them finish, to preserve existing behaviour.
 					api.Shutdown()
+					if dbConn != nil {
+						dbConn.Close()
+					}
 				},
 			})
 		},
diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 362c03001a..9a36128cde 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -2,6 +2,7 @@ package ledgerbackend
 
 import (
 	"context"
+	"encoding/hex"
 	"os"
 	"sync"
 
@@ -68,6 +69,7 @@ func (c *CaptiveStellarCore) roundDownToFirstReplayAfterCheckpointStart(ledger u
 type CaptiveStellarCore struct {
 	archive           historyarchive.ArchiveInterface
 	checkpointManager historyarchive.CheckpointManager
+	ledgerHashStore   TrustedLedgerHashStore
 
 	// cancel is the CancelFunc for context which controls the lifetime of a CaptiveStellarCore instance.
 	// Once it is invoked CaptiveStellarCore will not be able to stream ledgers from Stellar Core or
@@ -108,6 +110,8 @@ type CaptiveCoreConfig struct {
 	// CheckpointFrequency is the number of ledgers between checkpoints
 	// if unset, DefaultCheckpointFrequency will be used
 	CheckpointFrequency uint32
+	// LedgerHashStore is an optional store used to obtain hashes for ledger sequences from a trusted source
+	LedgerHashStore TrustedLedgerHashStore
 	// Log is an (optional) custom logger which will capture any output from the Stellar Core process.
 	// If Log is omitted then all output will be printed to stdout.
 	Log *log.Entry
@@ -120,6 +124,12 @@ type CaptiveCoreConfig struct {
 	// stored. We always append /captive-core to this directory, since we clean
 	// it up entirely on shutdown.
 	StoragePath string
+
+	// UseExternalStorageLedger, when true, instructs the core invocation to use an external db url
+	// for ledger states rather than in memory(RAM). The external db url is determined by the presence
+	// of DATABASE parameter in the captive-core-config-path or if absent, the db will default to sqlite
+	// and the db file will be stored at location derived from StoragePath parameter.
+	UseExternalStorageLedger bool
 }
 
 // NewCaptive returns a new CaptiveStellarCore instance.
@@ -158,6 +168,7 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 
 	c := &CaptiveStellarCore{
 		archive:           &archivePool,
+		ledgerHashStore:   config.LedgerHashStore,
 		cancel:            cancel,
 		checkpointManager: historyarchive.NewCheckpointManager(config.CheckpointFrequency),
 	}
@@ -249,12 +260,12 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	}
 	c.stellarCoreRunner = runner
 
-	runFrom, err := c.runFromParams(ctx, from)
+	runFrom, ledgerHash, err := c.runFromParams(ctx, from)
 	if err != nil {
 		return errors.Wrap(err, "error calculating ledger and hash for stellar-core run")
 	}
 
-	err = c.stellarCoreRunner.runFrom(runFrom)
+	err = c.stellarCoreRunner.runFrom(runFrom, ledgerHash)
 	if err != nil {
 		return errors.Wrap(err, "error running stellar-core")
 	}
@@ -271,14 +282,15 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 	return nil
 }
 
-// runFromParams receives a ledger sequence and calculates the required values to start stellar-core catchup from
-func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (runFrom uint32, err error) {
+// runFromParams receives a ledger sequence and calculates the required values to call stellar-core run with --start-ledger and --start-hash
+func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (runFrom uint32, ledgerHash string, err error) {
 	if from == 1 {
 		// Trying to start-from 1 results in an error from Stellar-Core:
 		// Target ledger 1 is not newer than last closed ledger 1 - nothing to do
 		// TODO maybe we can fix it by generating 1st ledger meta
 		// like GenesisLedgerStateReader?
-		return 0, errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
+		err = errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
+		return
 	}
 
 	if from <= 63 {
@@ -291,6 +303,24 @@ func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (ru
 	}
 
 	runFrom = from - 1
+	if c.ledgerHashStore != nil {
+		var exists bool
+		ledgerHash, exists, err = c.ledgerHashStore.GetLedgerHash(ctx, runFrom)
+		if err != nil {
+			err = errors.Wrapf(err, "error trying to read ledger hash %d", runFrom)
+			return
+		}
+		if exists {
+			return
+		}
+	}
+
+	ledgerHeader, err2 := c.archive.GetLedgerHeader(from)
+	if err2 != nil {
+		err = errors.Wrapf(err2, "error trying to read ledger header %d from HAS", from)
+		return
+	}
+	ledgerHash = hex.EncodeToString(ledgerHeader.Header.PreviousLedgerHash[:])
 	return
 }
 
@@ -616,6 +646,12 @@ func (c *CaptiveStellarCore) Close() error {
 	// after the CaptiveStellarCore context is canceled all subsequent calls to PrepareRange() will fail
 	c.cancel()
 
+	// TODO: Sucks to ignore the error here, but no worse than it was before,
+	// so...
+	if c.ledgerHashStore != nil {
+		c.ledgerHashStore.Close()
+	}
+
 	if c.stellarCoreRunner != nil {
 		return c.stellarCoreRunner.close()
 	}
diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
index a213551c02..f73341370f 100644
--- a/ingest/ledgerbackend/captive_core_backend_test.go
+++ b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -32,8 +32,8 @@ func (m *stellarCoreRunnerMock) catchup(from, to uint32) error {
 	return a.Error(0)
 }
 
-func (m *stellarCoreRunnerMock) runFrom(from uint32) error {
-	a := m.Called(from)
+func (m *stellarCoreRunnerMock) runFrom(from uint32, hash string) error {
+	a := m.Called(from, hash)
 	return a.Error(0)
 }
 
@@ -436,7 +436,7 @@ func TestCaptivePrepareRange_ErrCatchup(t *testing.T) {
 
 func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(127)).Return(errors.New("transient error")).Once()
+	mockRunner.On("runFrom", uint32(127), "0000000000000000000000000000000000000000000000000000000000000000").Return(errors.New("transient error")).Once()
 	mockRunner.On("close").Return(nil).Once()
 
 	mockArchive := &historyarchive.MockArchive{}
@@ -446,6 +446,10 @@ func TestCaptivePrepareRangeUnboundedRange_ErrRunFrom(t *testing.T) {
 			CurrentLedger: uint32(127),
 		}, nil)
 
+	mockArchive.
+		On("GetLedgerHeader", uint32(128)).
+		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
+
 	ctx := context.Background()
 	cancelCalled := false
 	captiveBackend := CaptiveStellarCore{
@@ -484,7 +488,7 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(64)).Return(nil).Once()
+	mockRunner.On("runFrom", uint32(64), "0000000000000000000000000000000000000000000000000000000000000000").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("getProcessExitError").Return(false, nil)
@@ -496,6 +500,10 @@ func TestCaptivePrepareRangeUnboundedRange_ReuseSession(t *testing.T) {
 			CurrentLedger: uint32(129),
 		}, nil)
 
+	mockArchive.
+		On("GetLedgerHeader", uint32(65)).
+		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
+
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -529,7 +537,7 @@ func TestGetLatestLedgerSequence(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(63)).Return(nil).Once()
+	mockRunner.On("runFrom", uint32(63), "0000000000000000000000000000000000000000000000000000000000000000").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -540,6 +548,10 @@ func TestGetLatestLedgerSequence(t *testing.T) {
 			CurrentLedger: uint32(200),
 		}, nil)
 
+	mockArchive.
+		On("GetLedgerHeader", uint32(64)).
+		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
+
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -664,7 +676,7 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(65)).Return(nil).Once()
+	mockRunner.On("runFrom", uint32(65), "0101010100000000000000000000000000000000000000000000000000000000").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 
@@ -675,6 +687,14 @@ func TestCaptiveGetLedgerCacheLatestLedger(t *testing.T) {
 			CurrentLedger: uint32(200),
 		}, nil)
 
+	mockArchive.
+		On("GetLedgerHeader", uint32(66)).
+		Return(xdr.LedgerHeaderHistoryEntry{
+			Header: xdr.LedgerHeader{
+				PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
+			},
+		}, nil).Once()
+
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -764,7 +784,7 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(64)).Return(nil)
+	mockRunner.On("runFrom", uint32(64), mock.Anything).Return(nil)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("close").Return(nil)
@@ -776,6 +796,10 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 			CurrentLedger: uint32(200),
 		}, nil)
 
+	mockArchive.
+		On("GetLedgerHeader", uint32(65)).
+		Return(xdr.LedgerHeaderHistoryEntry{}, nil)
+
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
@@ -1147,6 +1171,70 @@ func TestCaptiveGetLedgerTerminatedUnexpectedly(t *testing.T) {
 	}
 }
 
+func TestCaptiveUseOfLedgerHashStore(t *testing.T) {
+	ctx := context.Background()
+	mockArchive := &historyarchive.MockArchive{}
+	mockArchive.
+		On("GetLedgerHeader", uint32(300)).
+		Return(xdr.LedgerHeaderHistoryEntry{
+			Header: xdr.LedgerHeader{
+				PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
+			},
+		}, nil)
+
+	mockLedgerHashStore := &MockLedgerHashStore{}
+	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(1049)).
+		Return("", false, fmt.Errorf("transient error")).Once()
+	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(299)).
+		Return("", false, nil).Once()
+	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(85)).
+		Return("cde", true, nil).Once()
+	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(127)).
+		Return("ghi", true, nil).Once()
+	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(2)).
+		Return("mnb", true, nil).Once()
+
+	cancelCalled := false
+	captiveBackend := CaptiveStellarCore{
+		archive:           mockArchive,
+		ledgerHashStore:   mockLedgerHashStore,
+		checkpointManager: historyarchive.NewCheckpointManager(64),
+		cancel: context.CancelFunc(func() {
+			cancelCalled = true
+		}),
+	}
+
+	runFrom, ledgerHash, err := captiveBackend.runFromParams(ctx, 24)
+	assert.NoError(t, err)
+	assert.Equal(t, uint32(2), runFrom)
+	assert.Equal(t, "mnb", ledgerHash)
+
+	runFrom, ledgerHash, err = captiveBackend.runFromParams(ctx, 86)
+	assert.NoError(t, err)
+	assert.Equal(t, uint32(85), runFrom)
+	assert.Equal(t, "cde", ledgerHash)
+
+	runFrom, ledgerHash, err = captiveBackend.runFromParams(ctx, 128)
+	assert.NoError(t, err)
+	assert.Equal(t, uint32(127), runFrom)
+	assert.Equal(t, "ghi", ledgerHash)
+
+	_, _, err = captiveBackend.runFromParams(ctx, 1050)
+	assert.EqualError(t, err, "error trying to read ledger hash 1049: transient error")
+
+	runFrom, ledgerHash, err = captiveBackend.runFromParams(ctx, 300)
+	assert.NoError(t, err)
+	assert.Equal(t, uint32(299), runFrom, "runFrom")
+	assert.Equal(t, "0101010100000000000000000000000000000000000000000000000000000000", ledgerHash)
+
+	mockLedgerHashStore.On("Close").Return(nil).Once()
+	err = captiveBackend.Close()
+	assert.NoError(t, err)
+	assert.True(t, cancelCalled)
+	mockLedgerHashStore.AssertExpectations(t)
+	mockArchive.AssertExpectations(t)
+}
+
 func TestCaptiveRunFromParams(t *testing.T) {
 	var tests = []struct {
 		from           uint32
@@ -1177,6 +1265,13 @@ func TestCaptiveRunFromParams(t *testing.T) {
 		t.Run(fmt.Sprintf("from_%d", tc.from), func(t *testing.T) {
 			tt := assert.New(t)
 			mockArchive := &historyarchive.MockArchive{}
+			mockArchive.
+				On("GetLedgerHeader", uint32(tc.ledgerArchives)).
+				Return(xdr.LedgerHeaderHistoryEntry{
+					Header: xdr.LedgerHeader{
+						PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
+					},
+				}, nil)
 
 			captiveBackend := CaptiveStellarCore{
 				archive:           mockArchive,
@@ -1184,9 +1279,10 @@ func TestCaptiveRunFromParams(t *testing.T) {
 			}
 
 			ctx := context.Background()
-			runFrom, err := captiveBackend.runFromParams(ctx, tc.from)
+			runFrom, ledgerHash, err := captiveBackend.runFromParams(ctx, tc.from)
 			tt.NoError(err)
 			tt.Equal(tc.runFrom, runFrom, "runFrom")
+			tt.Equal("0101010100000000000000000000000000000000000000000000000000000000", ledgerHash)
 
 			mockArchive.AssertExpectations(t)
 		})
@@ -1311,7 +1407,7 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 
 	ctx := context.Background()
 	mockRunner := &stellarCoreRunnerMock{}
-	mockRunner.On("runFrom", uint32(299)).Return(nil).Once()
+	mockRunner.On("runFrom", uint32(299), "0101010100000000000000000000000000000000000000000000000000000000").Return(nil).Once()
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(ctx)
 	mockRunner.On("close").Return(nil).Once()
@@ -1322,12 +1418,24 @@ func TestCaptivePreviousLedgerCheck(t *testing.T) {
 		Return(historyarchive.HistoryArchiveState{
 			CurrentLedger: uint32(255),
 		}, nil)
+	mockArchive.
+		On("GetLedgerHeader", uint32(300)).
+		Return(xdr.LedgerHeaderHistoryEntry{
+			Header: xdr.LedgerHeader{
+				PreviousLedgerHash: xdr.Hash{1, 1, 1, 1},
+			},
+		}, nil).Once()
+
+	mockLedgerHashStore := &MockLedgerHashStore{}
+	mockLedgerHashStore.On("GetLedgerHash", ctx, uint32(299)).
+		Return("", false, nil).Once()
 
 	captiveBackend := CaptiveStellarCore{
 		archive: mockArchive,
 		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
 			return mockRunner, nil
 		},
+		ledgerHashStore:   mockLedgerHashStore,
 		checkpointManager: historyarchive.NewCheckpointManager(64),
 	}
 
diff --git a/ingest/ledgerbackend/ledger_hash_store.go b/ingest/ledgerbackend/ledger_hash_store.go
new file mode 100644
index 0000000000..be9ebf1bfc
--- /dev/null
+++ b/ingest/ledgerbackend/ledger_hash_store.go
@@ -0,0 +1,62 @@
+package ledgerbackend
+
+import (
+	"context"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/stretchr/testify/mock"
+
+	"github.com/stellar/go/support/db"
+)
+
+// TrustedLedgerHashStore is used to query ledger data from a trusted source.
+// The store should contain ledgers verified by Stellar-Core, do not use untrusted
+// source like history archives.
+type TrustedLedgerHashStore interface {
+	// GetLedgerHash returns the ledger hash for the given sequence number
+	GetLedgerHash(ctx context.Context, seq uint32) (string, bool, error)
+	Close() error
+}
+
+// HorizonDBLedgerHashStore is a TrustedLedgerHashStore which uses horizon's db to look up ledger hashes
+type HorizonDBLedgerHashStore struct {
+	session db.SessionInterface
+}
+
+// NewHorizonDBLedgerHashStore constructs a new TrustedLedgerHashStore backed by the horizon db
+func NewHorizonDBLedgerHashStore(session db.SessionInterface) TrustedLedgerHashStore {
+	return HorizonDBLedgerHashStore{session: session}
+}
+
+// GetLedgerHash returns the ledger hash for the given sequence number
+func (h HorizonDBLedgerHashStore) GetLedgerHash(ctx context.Context, seq uint32) (string, bool, error) {
+	sql := sq.Select("hl.ledger_hash").From("history_ledgers hl").
+		Limit(1).Where("sequence = ?", seq)
+
+	var hash string
+	err := h.session.Get(ctx, &hash, sql)
+	if h.session.NoRows(err) {
+		return hash, false, nil
+	}
+	return hash, true, err
+}
+
+func (h HorizonDBLedgerHashStore) Close() error {
+	return h.session.Close()
+}
+
+// MockLedgerHashStore is a mock implementation of TrustedLedgerHashStore
+type MockLedgerHashStore struct {
+	mock.Mock
+}
+
+// GetLedgerHash returns the ledger hash for the given sequence number
+func (m *MockLedgerHashStore) GetLedgerHash(ctx context.Context, seq uint32) (string, bool, error) {
+	args := m.Called(ctx, seq)
+	return args.Get(0).(string), args.Get(1).(bool), args.Error(2)
+}
+
+func (m *MockLedgerHashStore) Close() error {
+	args := m.Called()
+	return args.Error(0)
+}
diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index 4ebd791f5e..d9c3c93147 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -23,7 +23,7 @@ import (
 
 type stellarCoreRunnerInterface interface {
 	catchup(from, to uint32) error
-	runFrom(from uint32) error
+	runFrom(from uint32, hash string) error
 	getMetaPipe() <-chan metaResult
 	context() context.Context
 	getProcessExitError() (bool, error)
@@ -65,8 +65,9 @@ type stellarCoreRunner struct {
 	processExited    bool
 	processExitError error
 
-	storagePath string
-	nonce       string
+	storagePath              string
+	useExternalStorageLedger bool
+	nonce                    string
 
 	log *log.Entry
 }
@@ -118,11 +119,12 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 	ctx, cancel := context.WithCancel(config.Context)
 
 	runner := &stellarCoreRunner{
-		executablePath: config.BinaryPath,
-		ctx:            ctx,
-		cancel:         cancel,
-		storagePath:    fullStoragePath,
-		mode:           mode,
+		executablePath:           config.BinaryPath,
+		ctx:                      ctx,
+		cancel:                   cancel,
+		storagePath:              fullStoragePath,
+		useExternalStorageLedger: config.UseExternalStorageLedger,
+		mode:                     mode,
 		nonce: fmt.Sprintf(
 			"captive-stellar-core-%x",
 			rand.New(rand.NewSource(time.Now().UnixNano())).Uint64(),
@@ -130,7 +132,7 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 		log: config.Log,
 	}
 
-	if conf, err := writeConf(config.Toml, mode, runner.getConfFileName()); err != nil {
+	if conf, err := writeConf(config.Toml, mode, config, fullStoragePath, runner.getConfFileName()); err != nil {
 		return nil, errors.Wrap(err, "error writing configuration")
 	} else {
 		runner.log.Debugf("captive core config file contents:\n%s", conf)
@@ -139,8 +141,8 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 	return runner, nil
 }
 
-func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, location string) (string, error) {
-	text, err := generateConfig(captiveCoreToml, mode)
+func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig, fullStoragePath string, location string) (string, error) {
+	text, err := generateConfig(captiveCoreToml, mode, config, fullStoragePath)
 	if err != nil {
 		return "", err
 	}
@@ -148,9 +150,9 @@ func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, loc
 	return string(text), ioutil.WriteFile(location, text, 0644)
 }
 
-func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode) ([]byte, error) {
+func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig, fullStoragePath string) ([]byte, error) {
+	var err error
 	if mode == stellarCoreRunnerModeOffline {
-		var err error
 		captiveCoreToml, err = captiveCoreToml.CatchupToml()
 		if err != nil {
 			return nil, errors.Wrap(err, "could not generate catch up config")
@@ -161,6 +163,12 @@ func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode
 		return nil, errors.New("captive-core config file does not define any quorum set")
 	}
 
+	if config.UseExternalStorageLedger {
+		if captiveCoreToml, err = captiveCoreToml.ExternalLedgerStorageToml(fullStoragePath); err != nil {
+			return nil, errors.Wrap(err, "could not generate catch up config")
+		}
+	}
+
 	text, err := captiveCoreToml.Marshal()
 	if err != nil {
 		return nil, errors.Wrap(err, "could not marshal captive core config")
@@ -260,14 +268,25 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 		return errors.New("runner already started")
 	}
 
-	if err := r.createCmd("new-db").Run(); err != nil {
-		return errors.Wrap(err, "error initializing core db")
+	rangeArg := fmt.Sprintf("%d/%d", to, to-from+1)
+	inMemory := "--in-memory"
+
+	// horizon operator has specified to use external storage for captive core ledger state
+	// instruct captive core invocation to not use memory, and in that case
+	// cc will look at DATABASE property in cfg toml for the external storage source to use.
+	// when using external storage of ledgers, use new-db to first set the state of
+	// remote db storage to genesis to purge any prior state and reset.
+	if r.useExternalStorageLedger {
+		if err := r.createCmd("new-db").Run(); err != nil {
+			return errors.Wrap(err, "error initializing core db")
+		}
+		inMemory = ""
 	}
 
-	rangeArg := fmt.Sprintf("%d/%d", to, to-from+1)
 	r.cmd = r.createCmd(
 		"catchup", rangeArg,
 		"--metadata-output-stream", r.getPipeName(),
+		inMemory,
 	)
 
 	var err error
@@ -294,7 +313,7 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 }
 
 // runFrom executes the run command with a starting ledger on the captive core subprocess
-func (r *stellarCoreRunner) runFrom(from uint32) error {
+func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
@@ -307,29 +326,38 @@ func (r *stellarCoreRunner) runFrom(from uint32) error {
 		return errors.New("runner already started")
 	}
 
-	if err := r.createCmd("new-db").Run(); err != nil {
-		return errors.Wrap(err, "error initializing core db")
-	}
-
-	// Do a quick catch-up to set the LCL in core to be our expected starting
-	// point.
-	//
-	if from > 2 {
-		// Can't catch up to the genesis ledger.
-		if err := r.createCmd("catchup", fmt.Sprintf("%d/0", from-1)).Run(); err != nil {
-			return errors.Wrap(err, "error runing stellar-core catchup")
+	if r.useExternalStorageLedger {
+		if err := r.createCmd("new-db").Run(); err != nil {
+			return errors.Wrap(err, "error initializing core db")
 		}
-	} else {
-		if err := r.createCmd("catchup", "2/0").Run(); err != nil {
-			return errors.Wrap(err, "error runing stellar-core catchup")
+		// Do a quick catch-up to set the LCL in core to be our expected starting
+		// point.
+		//
+		if from > 2 {
+			// Can't catch up to the genesis ledger.
+			if err := r.createCmd("catchup", fmt.Sprintf("%d/0", from-1)).Run(); err != nil {
+				return errors.Wrap(err, "error runing stellar-core catchup")
+			}
+		} else {
+			if err := r.createCmd("catchup", "2/0").Run(); err != nil {
+				return errors.Wrap(err, "error runing stellar-core catchup")
+			}
 		}
+		r.cmd = r.createCmd(
+			"run",
+			"--metadata-output-stream",
+			r.getPipeName(),
+		)
+	} else {
+		r.cmd = r.createCmd(
+			"run",
+			"--in-memory",
+			"--start-at-ledger", fmt.Sprintf("%d", from),
+			"--start-at-hash", hash,
+			"--metadata-output-stream", r.getPipeName(),
+		)
 	}
 
-	r.cmd = r.createCmd(
-		"run",
-		"--metadata-output-stream", r.getPipeName(),
-	)
-
 	var err error
 	r.pipe, err = r.start(r.cmd)
 	if err != nil {
diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
index d6a80a628d..62aeeb6664 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
index 178764beab..124abc435b 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
index a0f54c6f70..d7b41a0b81 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
index e5acfe73e6..9eca1ccad1 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg
index 1fa85f8b84..57a5e7ff2c 100644
--- a/ingest/ledgerbackend/testdata/expected-online-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
index d164a8d67b..89e1762757 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
index 4ff5d3477a..1b65c5f318 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
index adc0dcf577..7797a14b5e 100644
--- a/ingest/ledgerbackend/toml.go
+++ b/ingest/ledgerbackend/toml.go
@@ -15,7 +15,6 @@ import (
 )
 
 const (
-	defaultDatabase      = "sqlite3://stellar.db"
 	defaultHTTPPort      = 11626
 	defaultFailureSafety = -1
 
@@ -31,6 +30,8 @@ var validQuality = map[string]bool{
 	"LOW":      true,
 }
 
+var sqliteUrlRegEx = regexp.MustCompile(`^sqlite3://\S+`)
+
 // Validator represents a [[VALIDATORS]] entry in the captive core toml file.
 type Validator struct {
 	Name       string `toml:"NAME"`
@@ -406,11 +407,21 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) {
 	return offline, nil
 }
 
-func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
-	if !c.tree.Has("DATABASE") {
-		c.Database = defaultDatabase
+func (c *CaptiveCoreToml) ExternalLedgerStorageToml(fullStoragePath string) (*CaptiveCoreToml, error) {
+	newToml, err := c.clone()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not clone toml")
 	}
 
+	if len(newToml.Database) == 0 {
+		newToml.Database = fmt.Sprintf("sqlite3://%v/stellar.db", fullStoragePath)
+	} else if !sqliteUrlRegEx.MatchString(newToml.Database) {
+		return nil, errors.Errorf("Invalid DATABASE parameter for captive core config, must be valid sqlite3 db url")
+	}
+	return newToml, nil
+}
+
+func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 	if !c.tree.Has("NETWORK_PASSPHRASE") {
 		c.NetworkPassphrase = params.NetworkPassphrase
 	}
diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go
index b8da4de03c..c66e21035b 100644
--- a/ingest/ledgerbackend/toml_test.go
+++ b/ingest/ledgerbackend/toml_test.go
@@ -308,7 +308,7 @@ func TestGenerateConfig(t *testing.T) {
 			}
 			assert.NoError(t, err)
 
-			configBytes, err := generateConfig(captiveCoreToml, testCase.mode)
+			configBytes, err := generateConfig(captiveCoreToml, testCase.mode, CaptiveCoreConfig{}, "")
 			assert.NoError(t, err)
 
 			expectedByte, err := ioutil.ReadFile(testCase.expectedPath)
@@ -318,3 +318,89 @@ func TestGenerateConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestExternalStorageConfigUsesDatabaseToml(t *testing.T) {
+	var err error
+	var captiveCoreToml *CaptiveCoreToml
+	httpPort := uint(8000)
+	peerPort := uint(8000)
+	logPath := "logPath"
+
+	params := CaptiveCoreTomlParams{
+		NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
+		HistoryArchiveURLs: []string{"http://localhost:1170"},
+		HTTPPort:           &httpPort,
+		PeerPort:           &peerPort,
+		LogPath:            &logPath,
+		Strict:             false,
+	}
+
+	captiveCoreToml, err = NewCaptiveCoreToml(params)
+	assert.NoError(t, err)
+	captiveCoreToml.Database = "sqlite3:///etc/defaults/stellar.db"
+
+	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
+		UseExternalStorageLedger: true,
+	}, "storagepath")
+
+	assert.NoError(t, err)
+	toml := CaptiveCoreToml{}
+	toml.unmarshal(configBytes, true)
+	assert.Equal(t, toml.Database, "sqlite3:///etc/defaults/stellar.db")
+}
+
+func TestExternalStorageConfigDefaultsToSqlite(t *testing.T) {
+	var err error
+	var captiveCoreToml *CaptiveCoreToml
+	httpPort := uint(8000)
+	peerPort := uint(8000)
+	logPath := "logPath"
+
+	params := CaptiveCoreTomlParams{
+		NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
+		HistoryArchiveURLs: []string{"http://localhost:1170"},
+		HTTPPort:           &httpPort,
+		PeerPort:           &peerPort,
+		LogPath:            &logPath,
+		Strict:             false,
+	}
+
+	captiveCoreToml, err = NewCaptiveCoreToml(params)
+	assert.NoError(t, err)
+
+	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
+		UseExternalStorageLedger: true,
+	}, "storagepath")
+
+	assert.NoError(t, err)
+	toml := CaptiveCoreToml{}
+	toml.unmarshal(configBytes, true)
+	assert.Equal(t, toml.Database, "sqlite3://storagepath/stellar.db")
+}
+
+func TestExternalStorageConfigRejectsNonSqliteDatabaseToml(t *testing.T) {
+	var err error
+	var captiveCoreToml *CaptiveCoreToml
+	httpPort := uint(8000)
+	peerPort := uint(8000)
+	logPath := "logPath"
+
+	params := CaptiveCoreTomlParams{
+		NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
+		HistoryArchiveURLs: []string{"http://localhost:1170"},
+		HTTPPort:           &httpPort,
+		PeerPort:           &peerPort,
+		LogPath:            &logPath,
+		Strict:             false,
+	}
+
+	captiveCoreToml, err = NewCaptiveCoreToml(params)
+	assert.NoError(t, err)
+	captiveCoreToml.Database = "baddb://whatever/stellar.db"
+
+	_, err = generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
+		UseExternalStorageLedger: true,
+	}, "storagepath")
+
+	assert.Error(t, err)
+}
diff --git a/integration.sh b/integration.sh
index 9ce0cb9b69..74978b38a1 100755
--- a/integration.sh
+++ b/integration.sh
@@ -6,6 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 export HORIZON_INTEGRATION_TESTS=true
 export HORIZON_INTEGRATION_ENABLE_CAP_35=${HORIZON_INTEGRATION_ENABLE_CAP_35:-}
 export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=${HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE:-}
+export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=${HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_REMOTE_STORAGE:-}
 export CAPTIVE_CORE_BIN=${CAPTIVE_CORE_BIN:-/usr/bin/stellar-core}
 export TRACY_NO_INVARIANT_CHECK=1 # This fails on my dev vm. - Paul
 
diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md
index c0d20f1f4f..5cd7896741 100644
--- a/services/horizon/CHANGELOG.md
+++ b/services/horizon/CHANGELOG.md
@@ -5,6 +5,8 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+* New feature, enable captive core based ingestion to use remote db persistence rather than in-memory for ledger states. Essentially moves what would have been stored in RAM to the external db instead. Recent profiling on the two approaches shows an approximate space usgae of about 8GB for ledger states as of 02/2022 timeframe, but it will gradually continue to increase as more accounts/assets are added to network. Current horizon ingest behavior when configured for captive core usage will by default take this space from RAM, unless a new command line flag is specified `--captive-core-use-external-storage=true`, which enables this space to be taken from the external db instead, and not RAM. The external db used is determined be setting `DATABASE` parameter in the captive core cfg/.toml file. If no value is set, then by default it uses sqlite and the db file is stored in `--captive-core-storage-path` - ([4092](https://github.com/stellar/go/pull/4092))
+
 ## v2.14.0
 
 * Restart Stellar-Core when it's context is cancelled. ([4192](https://github.com/stellar/go/pull/4192))
diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go
index d4f88b9aa7..458848436c 100644
--- a/services/horizon/cmd/db.go
+++ b/services/horizon/cmd/db.go
@@ -383,19 +383,20 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool,
 	}
 
 	ingestConfig := ingest.Config{
-		NetworkPassphrase:           config.NetworkPassphrase,
-		HistorySession:              horizonSession,
-		HistoryArchiveURL:           config.HistoryArchiveURLs[0],
-		CheckpointFrequency:         config.CheckpointFrequency,
-		MaxReingestRetries:          int(retries),
-		ReingestRetryBackoffSeconds: int(retryBackoffSeconds),
-		EnableCaptiveCore:           config.EnableCaptiveCoreIngestion,
-		CaptiveCoreBinaryPath:       config.CaptiveCoreBinaryPath,
-		RemoteCaptiveCoreURL:        config.RemoteCaptiveCoreURL,
-		CaptiveCoreToml:             config.CaptiveCoreToml,
-		CaptiveCoreStoragePath:      config.CaptiveCoreStoragePath,
-		StellarCoreCursor:           config.CursorName,
-		StellarCoreURL:              config.StellarCoreURL,
+		NetworkPassphrase:                         config.NetworkPassphrase,
+		HistorySession:                            horizonSession,
+		HistoryArchiveURL:                         config.HistoryArchiveURLs[0],
+		CheckpointFrequency:                       config.CheckpointFrequency,
+		MaxReingestRetries:                        int(retries),
+		ReingestRetryBackoffSeconds:               int(retryBackoffSeconds),
+		EnableCaptiveCore:                         config.EnableCaptiveCoreIngestion,
+		CaptiveCoreBinaryPath:                     config.CaptiveCoreBinaryPath,
+		CaptiveCoreConfigUseExternalStorageLedger: config.CaptiveCoreConfigUseExternalStorageLedger,
+		RemoteCaptiveCoreURL:                      config.RemoteCaptiveCoreURL,
+		CaptiveCoreToml:                           config.CaptiveCoreToml,
+		CaptiveCoreStoragePath:                    config.CaptiveCoreStoragePath,
+		StellarCoreCursor:                         config.CursorName,
+		StellarCoreURL:                            config.StellarCoreURL,
 	}
 
 	if !ingestConfig.EnableCaptiveCore {
diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go
index 2a7599921c..57d720447d 100644
--- a/services/horizon/cmd/ingest.go
+++ b/services/horizon/cmd/ingest.go
@@ -104,15 +104,16 @@ var ingestVerifyRangeCmd = &cobra.Command{
 		}
 
 		ingestConfig := ingest.Config{
-			NetworkPassphrase:      config.NetworkPassphrase,
-			HistorySession:         horizonSession,
-			HistoryArchiveURL:      config.HistoryArchiveURLs[0],
-			EnableCaptiveCore:      config.EnableCaptiveCoreIngestion,
-			CaptiveCoreBinaryPath:  config.CaptiveCoreBinaryPath,
-			RemoteCaptiveCoreURL:   config.RemoteCaptiveCoreURL,
-			CheckpointFrequency:    config.CheckpointFrequency,
-			CaptiveCoreToml:        config.CaptiveCoreToml,
-			CaptiveCoreStoragePath: config.CaptiveCoreStoragePath,
+			NetworkPassphrase:                         config.NetworkPassphrase,
+			HistorySession:                            horizonSession,
+			HistoryArchiveURL:                         config.HistoryArchiveURLs[0],
+			EnableCaptiveCore:                         config.EnableCaptiveCoreIngestion,
+			CaptiveCoreBinaryPath:                     config.CaptiveCoreBinaryPath,
+			CaptiveCoreConfigUseExternalStorageLedger: config.CaptiveCoreConfigUseExternalStorageLedger,
+			RemoteCaptiveCoreURL:                      config.RemoteCaptiveCoreURL,
+			CheckpointFrequency:                       config.CheckpointFrequency,
+			CaptiveCoreToml:                           config.CaptiveCoreToml,
+			CaptiveCoreStoragePath:                    config.CaptiveCoreStoragePath,
 		}
 
 		if !ingestConfig.EnableCaptiveCore {
@@ -206,6 +207,7 @@ var ingestStressTestCmd = &cobra.Command{
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath
 			ingestConfig.RemoteCaptiveCoreURL = config.RemoteCaptiveCoreURL
+			ingestConfig.CaptiveCoreConfigUseExternalStorageLedger = config.CaptiveCoreConfigUseExternalStorageLedger
 		} else {
 			if config.StellarCoreDatabaseURL == "" {
 				return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
@@ -295,6 +297,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{
 
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath
+			ingestConfig.CaptiveCoreConfigUseExternalStorageLedger = config.CaptiveCoreConfigUseExternalStorageLedger
 		} else {
 			if config.StellarCoreDatabaseURL == "" {
 				return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go
index b9e69d3b36..db2d13a0f9 100644
--- a/services/horizon/internal/config.go
+++ b/services/horizon/internal/config.go
@@ -19,15 +19,16 @@ type Config struct {
 	Port               uint
 	AdminPort          uint
 
-	EnableCaptiveCoreIngestion  bool
-	UsingDefaultPubnetConfig    bool
-	CaptiveCoreBinaryPath       string
-	RemoteCaptiveCoreURL        string
-	CaptiveCoreConfigPath       string
-	CaptiveCoreTomlParams       ledgerbackend.CaptiveCoreTomlParams
-	CaptiveCoreToml             *ledgerbackend.CaptiveCoreToml
-	CaptiveCoreStoragePath      string
-	CaptiveCoreReuseStoragePath bool
+	EnableCaptiveCoreIngestion                bool
+	UsingDefaultPubnetConfig                  bool
+	CaptiveCoreBinaryPath                     string
+	RemoteCaptiveCoreURL                      string
+	CaptiveCoreConfigPath                     string
+	CaptiveCoreTomlParams                     ledgerbackend.CaptiveCoreTomlParams
+	CaptiveCoreToml                           *ledgerbackend.CaptiveCoreToml
+	CaptiveCoreStoragePath                    string
+	CaptiveCoreReuseStoragePath               bool
+	CaptiveCoreConfigUseExternalStorageLedger bool
 
 	StellarCoreDatabaseURL string
 	StellarCoreURL         string
diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go
index 2d7d214631..d29b0ec522 100644
--- a/services/horizon/internal/flags.go
+++ b/services/horizon/internal/flags.go
@@ -36,6 +36,8 @@ const (
 	captiveCoreConfigAppendPathName = "captive-core-config-append-path"
 	// CaptiveCoreConfigPathName is the command line flag for configuring the path to the captive core configuration file
 	CaptiveCoreConfigPathName = "captive-core-config-path"
+	// captive-core-use-external-storage is the command line flag for enabling captive core runtime to use an external db url connection rather than RAM for ledger states
+	CaptiveCoreConfigUseExternalStorageLedger = "captive-core-use-external-storage"
 
 	captiveCoreMigrationHint = "If you are migrating from Horizon 1.x.y, start with the Migration Guide here: https://developers.stellar.org/docs/run-api-server/migrating/"
 )
@@ -170,6 +172,18 @@ func Flags() (*Config, support.ConfigOptions) {
 				return nil
 			},
 		},
+		&support.ConfigOption{
+			Name:        CaptiveCoreConfigUseExternalStorageLedger,
+			OptType:     types.Bool,
+			FlagDefault: false,
+			Required:    false,
+			Usage: `when enabled, Horizon ingestion will instruct the captive
+			              core invocation to use an external db url for ledger states rather than in memory(RAM).\n 
+						  Will result in several GB of space shifting out of RAM and to the external db persistence.\n
+						  The external db url is determined by the presence of DATABASE parameter in the captive-core-config-path or\n
+						  or if absent, the db will default to sqlite and the db file will be stored at location derived from captive-core-storage-path parameter.`,
+			ConfigKey: &config.CaptiveCoreConfigUseExternalStorageLedger,
+		},
 		&support.ConfigOption{
 			Name:        "enable-captive-core-ingestion",
 			OptType:     types.Bool,
diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go
index be30c1de4c..5f04082726 100644
--- a/services/horizon/internal/ingest/main.go
+++ b/services/horizon/internal/ingest/main.go
@@ -80,8 +80,9 @@ type Config struct {
 	HistorySession    db.SessionInterface
 	HistoryArchiveURL string
 
-	DisableStateVerification     bool
-	EnableExtendedLogLedgerStats bool
+	DisableStateVerification                  bool
+	EnableExtendedLogLedgerStats              bool
+	CaptiveCoreConfigUseExternalStorageLedger bool
 
 	MaxReingestRetries          int
 	ReingestRetryBackoffSeconds int
@@ -222,14 +223,15 @@ func NewSystem(config Config) (System, error) {
 			logger := log.WithField("subservice", "stellar-core")
 			ledgerBackend, err = ledgerbackend.NewCaptive(
 				ledgerbackend.CaptiveCoreConfig{
-					BinaryPath:          config.CaptiveCoreBinaryPath,
-					StoragePath:         config.CaptiveCoreStoragePath,
-					Toml:                config.CaptiveCoreToml,
-					NetworkPassphrase:   config.NetworkPassphrase,
-					HistoryArchiveURLs:  []string{config.HistoryArchiveURL},
-					CheckpointFrequency: config.CheckpointFrequency,
-					Log:                 logger,
-					Context:             ctx,
+					BinaryPath:               config.CaptiveCoreBinaryPath,
+					StoragePath:              config.CaptiveCoreStoragePath,
+					UseExternalStorageLedger: config.CaptiveCoreConfigUseExternalStorageLedger,
+					Toml:                     config.CaptiveCoreToml,
+					NetworkPassphrase:        config.NetworkPassphrase,
+					HistoryArchiveURLs:       []string{config.HistoryArchiveURL},
+					CheckpointFrequency:      config.CheckpointFrequency,
+					Log:                      logger,
+					Context:                  ctx,
 				},
 			)
 			if err != nil {
diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go
index 7b904bd30d..c459dcedde 100644
--- a/services/horizon/internal/init.go
+++ b/services/horizon/internal/init.go
@@ -89,17 +89,18 @@ func initIngester(app *App) {
 		// TODO:
 		// Use the first archive for now. We don't have a mechanism to
 		// use multiple archives at the same time currently.
-		HistoryArchiveURL:            app.config.HistoryArchiveURLs[0],
-		CheckpointFrequency:          app.config.CheckpointFrequency,
-		StellarCoreURL:               app.config.StellarCoreURL,
-		StellarCoreCursor:            app.config.CursorName,
-		CaptiveCoreBinaryPath:        app.config.CaptiveCoreBinaryPath,
-		CaptiveCoreStoragePath:       app.config.CaptiveCoreStoragePath,
-		CaptiveCoreToml:              app.config.CaptiveCoreToml,
-		RemoteCaptiveCoreURL:         app.config.RemoteCaptiveCoreURL,
-		EnableCaptiveCore:            app.config.EnableCaptiveCoreIngestion,
-		DisableStateVerification:     app.config.IngestDisableStateVerification,
-		EnableExtendedLogLedgerStats: app.config.IngestEnableExtendedLogLedgerStats,
+		HistoryArchiveURL:                         app.config.HistoryArchiveURLs[0],
+		CheckpointFrequency:                       app.config.CheckpointFrequency,
+		StellarCoreURL:                            app.config.StellarCoreURL,
+		StellarCoreCursor:                         app.config.CursorName,
+		CaptiveCoreBinaryPath:                     app.config.CaptiveCoreBinaryPath,
+		CaptiveCoreStoragePath:                    app.config.CaptiveCoreStoragePath,
+		CaptiveCoreConfigUseExternalStorageLedger: app.config.CaptiveCoreConfigUseExternalStorageLedger,
+		CaptiveCoreToml:                           app.config.CaptiveCoreToml,
+		RemoteCaptiveCoreURL:                      app.config.RemoteCaptiveCoreURL,
+		EnableCaptiveCore:                         app.config.EnableCaptiveCoreIngestion,
+		DisableStateVerification:                  app.config.IngestDisableStateVerification,
+		EnableExtendedLogLedgerStats:              app.config.IngestEnableExtendedLogLedgerStats,
 	})
 
 	if err != nil {
diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go
index 1d0a391327..82ca8c18b4 100644
--- a/services/horizon/internal/integration/db_test.go
+++ b/services/horizon/internal/integration/db_test.go
@@ -240,6 +240,8 @@ func command(horizonConfig horizon.Config, args ...string) []string {
 		horizonConfig.CaptiveCoreBinaryPath,
 		"--captive-core-config-path",
 		horizonConfig.CaptiveCoreConfigPath,
+		"--captive-core-use-external-storage=" +
+			strconv.FormatBool(horizonConfig.CaptiveCoreConfigUseExternalStorageLedger),
 		"--enable-captive-core-ingestion=" + strconv.FormatBool(horizonConfig.EnableCaptiveCoreIngestion),
 		"--network-passphrase",
 		horizonConfig.NetworkPassphrase,
diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go
index 51cb2eb1de..a66fe421dd 100644
--- a/services/horizon/internal/test/integration/integration.go
+++ b/services/horizon/internal/test/integration/integration.go
@@ -41,7 +41,8 @@ const (
 )
 
 var (
-	RunWithCaptiveCore = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE") != ""
+	RunWithCaptiveCore              = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE") != ""
+	RunWithCaptiveCoreRemoteStorage = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_REMOTE_STORAGE") != ""
 )
 
 type Config struct {
@@ -68,8 +69,9 @@ type Config struct {
 }
 
 type CaptiveConfig struct {
-	binaryPath string
-	configPath string
+	binaryPath       string
+	configPath       string
+	useRemoteStorage bool
 }
 
 type Test struct {
@@ -153,6 +155,9 @@ func (i *Test) configureCaptiveCore() {
 		composePath := findDockerComposePath()
 		i.coreConfig.binaryPath = os.Getenv("CAPTIVE_CORE_BIN")
 		i.coreConfig.configPath = filepath.Join(composePath, "captive-core-integration-tests.cfg")
+		if RunWithCaptiveCoreRemoteStorage {
+			i.coreConfig.useRemoteStorage = true
+		}
 	}
 
 	if value := i.getParameter(
@@ -294,6 +299,7 @@ func (i *Test) StartHorizon() error {
 	hostname := "localhost"
 	coreBinaryPath := i.coreConfig.binaryPath
 	captiveCoreConfigPath := i.coreConfig.configPath
+	captiveCoreUseRemoteStorage := strconv.FormatBool(i.coreConfig.useRemoteStorage)
 
 	defaultArgs := map[string]string{
 		"stellar-core-url": i.coreClient.URL,
@@ -303,17 +309,18 @@ func (i *Test) StartHorizon() error {
 			hostname,
 			stellarCorePostgresPort,
 		),
-		"stellar-core-binary-path":      coreBinaryPath,
-		"captive-core-config-path":      captiveCoreConfigPath,
-		"captive-core-http-port":        "21626",
-		"enable-captive-core-ingestion": strconv.FormatBool(len(coreBinaryPath) > 0),
-		"ingest":                        "true",
-		"history-archive-urls":          fmt.Sprintf("http://%s:%d", hostname, historyArchivePort),
-		"db-url":                        horizonPostgresURL,
-		"network-passphrase":            i.passPhrase,
-		"apply-migrations":              "true",
-		"admin-port":                    strconv.Itoa(i.AdminPort()),
-		"port":                          "8000",
+		"stellar-core-binary-path":          coreBinaryPath,
+		"captive-core-config-path":          captiveCoreConfigPath,
+		"captive-core-http-port":            "21626",
+		"captive-core-use-external-storage": captiveCoreUseRemoteStorage,
+		"enable-captive-core-ingestion":     strconv.FormatBool(len(coreBinaryPath) > 0),
+		"ingest":                            "true",
+		"history-archive-urls":              fmt.Sprintf("http://%s:%d", hostname, historyArchivePort),
+		"db-url":                            horizonPostgresURL,
+		"network-passphrase":                i.passPhrase,
+		"apply-migrations":                  "true",
+		"admin-port":                        strconv.Itoa(i.AdminPort()),
+		"port":                              "8000",
 		// due to ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING
 		"checkpoint-frequency": "8",
 		"per-hour-rate-limit":  "0", // disable rate limiting

From 21ca9cf2823139fe5a7ef7c8bc93d24b84877fe3 Mon Sep 17 00:00:00 2001
From: shawn <sreuland@users.noreply.github.com>
Date: Wed, 16 Feb 2022 12:43:41 -0800
Subject: [PATCH 15/17] 4038: incorporate PR feedback updates

---
 ingest/ledgerbackend/captive_core_backend.go  | 20 +++++++++++--------
 ingest/ledgerbackend/stellar_core_runner.go   |  1 -
 .../internal/db2/history/ledger_test.go       | 11 +++++-----
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 9a36128cde..7ab8cbbc5f 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -283,14 +283,19 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 }
 
 // runFromParams receives a ledger sequence and calculates the required values to call stellar-core run with --start-ledger and --start-hash
-func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (runFrom uint32, ledgerHash string, err error) {
+func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (uint32, string, error) {
+
+	var runFrom uint32
+	var ledgerHash string
+	var err error
+
 	if from == 1 {
 		// Trying to start-from 1 results in an error from Stellar-Core:
 		// Target ledger 1 is not newer than last closed ledger 1 - nothing to do
 		// TODO maybe we can fix it by generating 1st ledger meta
 		// like GenesisLedgerStateReader?
 		err = errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
-		return
+		return 0, "", err
 	}
 
 	if from <= 63 {
@@ -308,20 +313,20 @@ func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (ru
 		ledgerHash, exists, err = c.ledgerHashStore.GetLedgerHash(ctx, runFrom)
 		if err != nil {
 			err = errors.Wrapf(err, "error trying to read ledger hash %d", runFrom)
-			return
+			return 0, "", err
 		}
 		if exists {
-			return
+			return runFrom, ledgerHash, nil
 		}
 	}
 
 	ledgerHeader, err2 := c.archive.GetLedgerHeader(from)
 	if err2 != nil {
 		err = errors.Wrapf(err2, "error trying to read ledger header %d from HAS", from)
-		return
+		return 0, "", err
 	}
 	ledgerHash = hex.EncodeToString(ledgerHeader.Header.PreviousLedgerHash[:])
-	return
+	return runFrom, ledgerHash, nil
 }
 
 // nextExpectedSequence returns nextLedger (if currently set) or start of
@@ -410,8 +415,7 @@ func (c *CaptiveStellarCore) isPrepared(ledgerRange Range) bool {
 		return false
 	}
 
-	exited, _ := c.stellarCoreRunner.getProcessExitError()
-	if exited {
+	if exited, _ := c.stellarCoreRunner.getProcessExitError(); exited {
 		return false
 	}
 
diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index d9c3c93147..ff50260a84 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -332,7 +332,6 @@ func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
 		}
 		// Do a quick catch-up to set the LCL in core to be our expected starting
 		// point.
-		//
 		if from > 2 {
 			// Can't catch up to the genesis ledger.
 			if err := r.createCmd("catchup", fmt.Sprintf("%d/0", from-1)).Run(); err != nil {
diff --git a/services/horizon/internal/db2/history/ledger_test.go b/services/horizon/internal/db2/history/ledger_test.go
index b5d5651cff..4bf6d7b058 100644
--- a/services/horizon/internal/db2/history/ledger_test.go
+++ b/services/horizon/internal/db2/history/ledger_test.go
@@ -60,7 +60,11 @@ func TestInsertLedger(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	// TODO: Check the ledger doesn't exist in the db
+	var ledgerFromDB Ledger
+	var ledgerHeaderBase64 string
+	var err error
+	err = q.LedgerBySequence(tt.Ctx, &ledgerFromDB, 69859)
+	tt.Assert.Error(err)
 
 	expectedLedger := Ledger{
 		Sequence:                   69859,
@@ -111,7 +115,7 @@ func TestInsertLedger(t *testing.T) {
 			},
 		},
 	}
-	ledgerHeaderBase64, err := xdr.MarshalBase64(ledgerEntry.Header)
+	ledgerHeaderBase64, err = xdr.MarshalBase64(ledgerEntry.Header)
 	tt.Assert.NoError(err)
 	expectedLedger.LedgerHeaderXDR = null.NewString(ledgerHeaderBase64, true)
 
@@ -126,7 +130,6 @@ func TestInsertLedger(t *testing.T) {
 	tt.Assert.NoError(err)
 	tt.Assert.Equal(rowsAffected, int64(1))
 
-	var ledgerFromDB Ledger
 	err = q.LedgerBySequence(tt.Ctx, &ledgerFromDB, 69859)
 	tt.Assert.NoError(err)
 
@@ -141,8 +144,6 @@ func TestInsertLedger(t *testing.T) {
 	expectedLedger.ClosedAt = ledgerFromDB.ClosedAt
 
 	tt.Assert.Equal(expectedLedger, ledgerFromDB)
-
-	// TODO: Check the ledger does exist in the db
 }
 
 func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) {

From 5185fff920083af051231bd966bdbab3127cd2d0 Mon Sep 17 00:00:00 2001
From: shawn <sreuland@users.noreply.github.com>
Date: Wed, 16 Feb 2022 20:50:01 -0800
Subject: [PATCH 16/17] #4038: incorporate PR feedback, rename the sqlite db
 parameter to UseDB

---
 .circleci/config.yml                          |  2 +-
 ingest/ledgerbackend/captive_core_backend.go  | 21 ++++----
 ingest/ledgerbackend/stellar_core_runner.go   | 36 ++++++--------
 .../testdata/expected-offline-core.cfg        |  1 +
 .../expected-offline-with-appendix-core.cfg   |  1 +
 .../expected-offline-with-extra-fields.cfg    |  1 +
 .../expected-offline-with-no-peer-port.cfg    |  1 +
 .../testdata/expected-online-core.cfg         |  1 +
 .../expected-online-with-no-http-port.cfg     |  1 +
 .../expected-online-with-no-peer-port.cfg     |  1 +
 .../invalid-captive-core-database-field.cfg   | 12 +++++
 ingest/ledgerbackend/toml.go                  | 21 +++-----
 ingest/ledgerbackend/toml_test.go             | 48 ++++++-------------
 integration.sh                                |  2 +-
 services/horizon/CHANGELOG.md                 |  2 +-
 services/horizon/cmd/db.go                    | 28 +++++------
 services/horizon/cmd/ingest.go                | 24 +++++-----
 services/horizon/internal/config.go           | 20 ++++----
 services/horizon/internal/flags.go            |  8 ++--
 services/horizon/internal/ingest/main.go      | 24 +++++-----
 services/horizon/internal/init.go             | 24 +++++-----
 .../horizon/internal/integration/db_test.go   |  4 +-
 .../internal/test/integration/integration.go  | 40 ++++++++--------
 23 files changed, 153 insertions(+), 170 deletions(-)
 create mode 100644 ingest/ledgerbackend/testdata/invalid-captive-core-database-field.cfg

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a0c175c7fb..9ac4b8c1f8 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -460,7 +460,7 @@ jobs:
           steps:
             - run:
                 name: Setting Captive Core Remote Storage env variable
-                command: echo "export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_REMOTE_STORAGE=true" >> $BASH_ENV          
+                command: echo "export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_USE_DB=true" >> $BASH_ENV          
       - run:
           name: Run Horizon integration tests <<#parameters.enable-captive-core>>(With captive core)<</parameters.enable-captive-core>>
           # Currently all integration tests are in a single directory.
diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index 7ab8cbbc5f..fad024589d 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -129,7 +129,7 @@ type CaptiveCoreConfig struct {
 	// for ledger states rather than in memory(RAM). The external db url is determined by the presence
 	// of DATABASE parameter in the captive-core-config-path or if absent, the db will default to sqlite
 	// and the db file will be stored at location derived from StoragePath parameter.
-	UseExternalStorageLedger bool
+	UseDB bool
 }
 
 // NewCaptive returns a new CaptiveStellarCore instance.
@@ -285,16 +285,12 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(ctx context.Context, fro
 // runFromParams receives a ledger sequence and calculates the required values to call stellar-core run with --start-ledger and --start-hash
 func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (uint32, string, error) {
 
-	var runFrom uint32
-	var ledgerHash string
-	var err error
-
 	if from == 1 {
 		// Trying to start-from 1 results in an error from Stellar-Core:
 		// Target ledger 1 is not newer than last closed ledger 1 - nothing to do
 		// TODO maybe we can fix it by generating 1st ledger meta
 		// like GenesisLedgerStateReader?
-		err = errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
+		err := errors.New("CaptiveCore is unable to start from ledger 1, start from ledger 2")
 		return 0, "", err
 	}
 
@@ -307,10 +303,10 @@ func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (ui
 		from = 3
 	}
 
-	runFrom = from - 1
+	runFrom := from - 1
 	if c.ledgerHashStore != nil {
 		var exists bool
-		ledgerHash, exists, err = c.ledgerHashStore.GetLedgerHash(ctx, runFrom)
+		ledgerHash, exists, err := c.ledgerHashStore.GetLedgerHash(ctx, runFrom)
 		if err != nil {
 			err = errors.Wrapf(err, "error trying to read ledger hash %d", runFrom)
 			return 0, "", err
@@ -320,12 +316,11 @@ func (c *CaptiveStellarCore) runFromParams(ctx context.Context, from uint32) (ui
 		}
 	}
 
-	ledgerHeader, err2 := c.archive.GetLedgerHeader(from)
-	if err2 != nil {
-		err = errors.Wrapf(err2, "error trying to read ledger header %d from HAS", from)
-		return 0, "", err
+	ledgerHeader, err := c.archive.GetLedgerHeader(from)
+	if err != nil {
+		return 0, "", errors.Wrapf(err, "error trying to read ledger header %d from HAS", from)
 	}
-	ledgerHash = hex.EncodeToString(ledgerHeader.Header.PreviousLedgerHash[:])
+	ledgerHash := hex.EncodeToString(ledgerHeader.Header.PreviousLedgerHash[:])
 	return runFrom, ledgerHash, nil
 }
 
diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index ff50260a84..0fbac765ce 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -65,9 +65,9 @@ type stellarCoreRunner struct {
 	processExited    bool
 	processExitError error
 
-	storagePath              string
-	useExternalStorageLedger bool
-	nonce                    string
+	storagePath string
+	useDB       bool
+	nonce       string
 
 	log *log.Entry
 }
@@ -119,12 +119,12 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 	ctx, cancel := context.WithCancel(config.Context)
 
 	runner := &stellarCoreRunner{
-		executablePath:           config.BinaryPath,
-		ctx:                      ctx,
-		cancel:                   cancel,
-		storagePath:              fullStoragePath,
-		useExternalStorageLedger: config.UseExternalStorageLedger,
-		mode:                     mode,
+		executablePath: config.BinaryPath,
+		ctx:            ctx,
+		cancel:         cancel,
+		storagePath:    fullStoragePath,
+		useDB:          config.UseDB,
+		mode:           mode,
 		nonce: fmt.Sprintf(
 			"captive-stellar-core-%x",
 			rand.New(rand.NewSource(time.Now().UnixNano())).Uint64(),
@@ -132,7 +132,7 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 		log: config.Log,
 	}
 
-	if conf, err := writeConf(config.Toml, mode, config, fullStoragePath, runner.getConfFileName()); err != nil {
+	if conf, err := writeConf(config.Toml, mode, config, runner.getConfFileName()); err != nil {
 		return nil, errors.Wrap(err, "error writing configuration")
 	} else {
 		runner.log.Debugf("captive core config file contents:\n%s", conf)
@@ -141,8 +141,8 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 	return runner, nil
 }
 
-func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig, fullStoragePath string, location string) (string, error) {
-	text, err := generateConfig(captiveCoreToml, mode, config, fullStoragePath)
+func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig, location string) (string, error) {
+	text, err := generateConfig(captiveCoreToml, mode, config)
 	if err != nil {
 		return "", err
 	}
@@ -150,7 +150,7 @@ func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, con
 	return string(text), ioutil.WriteFile(location, text, 0644)
 }
 
-func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig, fullStoragePath string) ([]byte, error) {
+func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig) ([]byte, error) {
 	var err error
 	if mode == stellarCoreRunnerModeOffline {
 		captiveCoreToml, err = captiveCoreToml.CatchupToml()
@@ -163,12 +163,6 @@ func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode
 		return nil, errors.New("captive-core config file does not define any quorum set")
 	}
 
-	if config.UseExternalStorageLedger {
-		if captiveCoreToml, err = captiveCoreToml.ExternalLedgerStorageToml(fullStoragePath); err != nil {
-			return nil, errors.Wrap(err, "could not generate catch up config")
-		}
-	}
-
 	text, err := captiveCoreToml.Marshal()
 	if err != nil {
 		return nil, errors.Wrap(err, "could not marshal captive core config")
@@ -276,7 +270,7 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 	// cc will look at DATABASE property in cfg toml for the external storage source to use.
 	// when using external storage of ledgers, use new-db to first set the state of
 	// remote db storage to genesis to purge any prior state and reset.
-	if r.useExternalStorageLedger {
+	if r.useDB {
 		if err := r.createCmd("new-db").Run(); err != nil {
 			return errors.Wrap(err, "error initializing core db")
 		}
@@ -326,7 +320,7 @@ func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
 		return errors.New("runner already started")
 	}
 
-	if r.useExternalStorageLedger {
+	if r.useDB {
 		if err := r.createCmd("new-db").Run(); err != nil {
 			return errors.Wrap(err, "error initializing core db")
 		}
diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
index 62aeeb6664..d6a80a628d 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
index 124abc435b..178764beab 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
index d7b41a0b81..a0f54c6f70 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
index 9eca1ccad1..e5acfe73e6 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg
index 57a5e7ff2c..1fa85f8b84 100644
--- a/ingest/ledgerbackend/testdata/expected-online-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
index 89e1762757..d164a8d67b 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
index 1b65c5f318..4ff5d3477a 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/testdata/invalid-captive-core-database-field.cfg b/ingest/ledgerbackend/testdata/invalid-captive-core-database-field.cfg
new file mode 100644
index 0000000000..438bd99120
--- /dev/null
+++ b/ingest/ledgerbackend/testdata/invalid-captive-core-database-field.cfg
@@ -0,0 +1,12 @@
+# DATABASE limited to only be sqlite:// protocol 
+DATABASE="postgres://mydb"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="MEDIUM"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="localhost:123"
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
index 7797a14b5e..3935bee235 100644
--- a/ingest/ledgerbackend/toml.go
+++ b/ingest/ledgerbackend/toml.go
@@ -30,8 +30,6 @@ var validQuality = map[string]bool{
 	"LOW":      true,
 }
 
-var sqliteUrlRegEx = regexp.MustCompile(`^sqlite3://\S+`)
-
 // Validator represents a [[VALIDATORS]] entry in the captive core toml file.
 type Validator struct {
 	Name       string `toml:"NAME"`
@@ -407,21 +405,12 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) {
 	return offline, nil
 }
 
-func (c *CaptiveCoreToml) ExternalLedgerStorageToml(fullStoragePath string) (*CaptiveCoreToml, error) {
-	newToml, err := c.clone()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not clone toml")
-	}
+func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 
-	if len(newToml.Database) == 0 {
-		newToml.Database = fmt.Sprintf("sqlite3://%v/stellar.db", fullStoragePath)
-	} else if !sqliteUrlRegEx.MatchString(newToml.Database) {
-		return nil, errors.Errorf("Invalid DATABASE parameter for captive core config, must be valid sqlite3 db url")
+	if !c.tree.Has("DATABASE") {
+		c.Database = "sqlite3://stellar.db"
 	}
-	return newToml, nil
-}
 
-func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 	if !c.tree.Has("NETWORK_PASSPHRASE") {
 		c.NetworkPassphrase = params.NetworkPassphrase
 	}
@@ -566,5 +555,9 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error {
 		names[v.Name] = true
 	}
 
+	if len(c.Database) > 0 && !strings.HasPrefix(c.Database, "sqlite3://") {
+		return fmt.Errorf("invalid DATABASE parameter: %s, for captive core config, must be valid sqlite3 db url", c.Database)
+	}
+
 	return nil
 }
diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go
index c66e21035b..0465543a0c 100644
--- a/ingest/ledgerbackend/toml_test.go
+++ b/ingest/ledgerbackend/toml_test.go
@@ -186,6 +186,15 @@ func TestCaptiveCoreTomlValidation(t *testing.T) {
 			logPath:           nil,
 			expectedError:     "could not unmarshal captive core toml: these fields are not supported by captive core: [\"CATCHUP_RECENT\"]",
 		},
+		{
+			name:              "database field was invalid for captive core",
+			networkPassphrase: "Public Global Stellar Network ; September 2015",
+			appendPath:        filepath.Join("testdata", "invalid-captive-core-database-field.cfg"),
+			httpPort:          nil,
+			peerPort:          nil,
+			logPath:           nil,
+			expectedError:     `invalid captive core toml: invalid DATABASE parameter: postgres://mydb, for captive core config, must be valid sqlite3 db url`,
+		},
 		{
 			name:          "unexpected BUCKET_DIR_PATH",
 			appendPath:    filepath.Join("testdata", "appendix-with-bucket-dir-path.cfg"),
@@ -308,7 +317,7 @@ func TestGenerateConfig(t *testing.T) {
 			}
 			assert.NoError(t, err)
 
-			configBytes, err := generateConfig(captiveCoreToml, testCase.mode, CaptiveCoreConfig{}, "")
+			configBytes, err := generateConfig(captiveCoreToml, testCase.mode, CaptiveCoreConfig{})
 			assert.NoError(t, err)
 
 			expectedByte, err := ioutil.ReadFile(testCase.expectedPath)
@@ -340,8 +349,8 @@ func TestExternalStorageConfigUsesDatabaseToml(t *testing.T) {
 	captiveCoreToml.Database = "sqlite3:///etc/defaults/stellar.db"
 
 	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
-		UseExternalStorageLedger: true,
-	}, "storagepath")
+		UseDB: true,
+	})
 
 	assert.NoError(t, err)
 	toml := CaptiveCoreToml{}
@@ -369,38 +378,11 @@ func TestExternalStorageConfigDefaultsToSqlite(t *testing.T) {
 	assert.NoError(t, err)
 
 	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
-		UseExternalStorageLedger: true,
-	}, "storagepath")
+		UseDB: true,
+	})
 
 	assert.NoError(t, err)
 	toml := CaptiveCoreToml{}
 	toml.unmarshal(configBytes, true)
-	assert.Equal(t, toml.Database, "sqlite3://storagepath/stellar.db")
-}
-
-func TestExternalStorageConfigRejectsNonSqliteDatabaseToml(t *testing.T) {
-	var err error
-	var captiveCoreToml *CaptiveCoreToml
-	httpPort := uint(8000)
-	peerPort := uint(8000)
-	logPath := "logPath"
-
-	params := CaptiveCoreTomlParams{
-		NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
-		HistoryArchiveURLs: []string{"http://localhost:1170"},
-		HTTPPort:           &httpPort,
-		PeerPort:           &peerPort,
-		LogPath:            &logPath,
-		Strict:             false,
-	}
-
-	captiveCoreToml, err = NewCaptiveCoreToml(params)
-	assert.NoError(t, err)
-	captiveCoreToml.Database = "baddb://whatever/stellar.db"
-
-	_, err = generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
-		UseExternalStorageLedger: true,
-	}, "storagepath")
-
-	assert.Error(t, err)
+	assert.Equal(t, toml.Database, "sqlite3://stellar.db")
 }
diff --git a/integration.sh b/integration.sh
index 74978b38a1..1bd78a5c16 100755
--- a/integration.sh
+++ b/integration.sh
@@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 export HORIZON_INTEGRATION_TESTS=true
 export HORIZON_INTEGRATION_ENABLE_CAP_35=${HORIZON_INTEGRATION_ENABLE_CAP_35:-}
 export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=${HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE:-}
-export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=${HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_REMOTE_STORAGE:-}
+export HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_USE_DB=${HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_USE_DB:-}
 export CAPTIVE_CORE_BIN=${CAPTIVE_CORE_BIN:-/usr/bin/stellar-core}
 export TRACY_NO_INVARIANT_CHECK=1 # This fails on my dev vm. - Paul
 
diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md
index 5cd7896741..87c2cd42b3 100644
--- a/services/horizon/CHANGELOG.md
+++ b/services/horizon/CHANGELOG.md
@@ -5,7 +5,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
-* New feature, enable captive core based ingestion to use remote db persistence rather than in-memory for ledger states. Essentially moves what would have been stored in RAM to the external db instead. Recent profiling on the two approaches shows an approximate space usgae of about 8GB for ledger states as of 02/2022 timeframe, but it will gradually continue to increase as more accounts/assets are added to network. Current horizon ingest behavior when configured for captive core usage will by default take this space from RAM, unless a new command line flag is specified `--captive-core-use-external-storage=true`, which enables this space to be taken from the external db instead, and not RAM. The external db used is determined be setting `DATABASE` parameter in the captive core cfg/.toml file. If no value is set, then by default it uses sqlite and the db file is stored in `--captive-core-storage-path` - ([4092](https://github.com/stellar/go/pull/4092))
+* New feature, enable captive core based ingestion to use remote db persistence rather than in-memory for ledger states. Essentially moves what would have been stored in RAM to the external db instead. Recent profiling on the two approaches shows an approximate space usgae of about 8GB for ledger states as of 02/2022 timeframe, but it will gradually continue to increase as more accounts/assets are added to network. Current horizon ingest behavior when configured for captive core usage will by default take this space from RAM, unless a new command line flag is specified `--captive-core-use-db=true`, which enables this space to be taken from the external db instead, and not RAM. The external db used is determined be setting `DATABASE` parameter in the captive core cfg/.toml file. If no value is set, then by default it uses sqlite and the db file is stored in `--captive-core-storage-path` - ([4092](https://github.com/stellar/go/pull/4092))
 
 ## v2.14.0
 
diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go
index 458848436c..37912f9e76 100644
--- a/services/horizon/cmd/db.go
+++ b/services/horizon/cmd/db.go
@@ -383,20 +383,20 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool,
 	}
 
 	ingestConfig := ingest.Config{
-		NetworkPassphrase:                         config.NetworkPassphrase,
-		HistorySession:                            horizonSession,
-		HistoryArchiveURL:                         config.HistoryArchiveURLs[0],
-		CheckpointFrequency:                       config.CheckpointFrequency,
-		MaxReingestRetries:                        int(retries),
-		ReingestRetryBackoffSeconds:               int(retryBackoffSeconds),
-		EnableCaptiveCore:                         config.EnableCaptiveCoreIngestion,
-		CaptiveCoreBinaryPath:                     config.CaptiveCoreBinaryPath,
-		CaptiveCoreConfigUseExternalStorageLedger: config.CaptiveCoreConfigUseExternalStorageLedger,
-		RemoteCaptiveCoreURL:                      config.RemoteCaptiveCoreURL,
-		CaptiveCoreToml:                           config.CaptiveCoreToml,
-		CaptiveCoreStoragePath:                    config.CaptiveCoreStoragePath,
-		StellarCoreCursor:                         config.CursorName,
-		StellarCoreURL:                            config.StellarCoreURL,
+		NetworkPassphrase:           config.NetworkPassphrase,
+		HistorySession:              horizonSession,
+		HistoryArchiveURL:           config.HistoryArchiveURLs[0],
+		CheckpointFrequency:         config.CheckpointFrequency,
+		MaxReingestRetries:          int(retries),
+		ReingestRetryBackoffSeconds: int(retryBackoffSeconds),
+		EnableCaptiveCore:           config.EnableCaptiveCoreIngestion,
+		CaptiveCoreBinaryPath:       config.CaptiveCoreBinaryPath,
+		CaptiveCoreConfigUseDB:      config.CaptiveCoreConfigUseDB,
+		RemoteCaptiveCoreURL:        config.RemoteCaptiveCoreURL,
+		CaptiveCoreToml:             config.CaptiveCoreToml,
+		CaptiveCoreStoragePath:      config.CaptiveCoreStoragePath,
+		StellarCoreCursor:           config.CursorName,
+		StellarCoreURL:              config.StellarCoreURL,
 	}
 
 	if !ingestConfig.EnableCaptiveCore {
diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go
index 57d720447d..b0e0f19a10 100644
--- a/services/horizon/cmd/ingest.go
+++ b/services/horizon/cmd/ingest.go
@@ -104,16 +104,16 @@ var ingestVerifyRangeCmd = &cobra.Command{
 		}
 
 		ingestConfig := ingest.Config{
-			NetworkPassphrase:                         config.NetworkPassphrase,
-			HistorySession:                            horizonSession,
-			HistoryArchiveURL:                         config.HistoryArchiveURLs[0],
-			EnableCaptiveCore:                         config.EnableCaptiveCoreIngestion,
-			CaptiveCoreBinaryPath:                     config.CaptiveCoreBinaryPath,
-			CaptiveCoreConfigUseExternalStorageLedger: config.CaptiveCoreConfigUseExternalStorageLedger,
-			RemoteCaptiveCoreURL:                      config.RemoteCaptiveCoreURL,
-			CheckpointFrequency:                       config.CheckpointFrequency,
-			CaptiveCoreToml:                           config.CaptiveCoreToml,
-			CaptiveCoreStoragePath:                    config.CaptiveCoreStoragePath,
+			NetworkPassphrase:      config.NetworkPassphrase,
+			HistorySession:         horizonSession,
+			HistoryArchiveURL:      config.HistoryArchiveURLs[0],
+			EnableCaptiveCore:      config.EnableCaptiveCoreIngestion,
+			CaptiveCoreBinaryPath:  config.CaptiveCoreBinaryPath,
+			CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB,
+			RemoteCaptiveCoreURL:   config.RemoteCaptiveCoreURL,
+			CheckpointFrequency:    config.CheckpointFrequency,
+			CaptiveCoreToml:        config.CaptiveCoreToml,
+			CaptiveCoreStoragePath: config.CaptiveCoreStoragePath,
 		}
 
 		if !ingestConfig.EnableCaptiveCore {
@@ -207,7 +207,7 @@ var ingestStressTestCmd = &cobra.Command{
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath
 			ingestConfig.RemoteCaptiveCoreURL = config.RemoteCaptiveCoreURL
-			ingestConfig.CaptiveCoreConfigUseExternalStorageLedger = config.CaptiveCoreConfigUseExternalStorageLedger
+			ingestConfig.CaptiveCoreConfigUseDB = config.CaptiveCoreConfigUseDB
 		} else {
 			if config.StellarCoreDatabaseURL == "" {
 				return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
@@ -297,7 +297,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{
 
 		if config.EnableCaptiveCoreIngestion {
 			ingestConfig.CaptiveCoreBinaryPath = config.CaptiveCoreBinaryPath
-			ingestConfig.CaptiveCoreConfigUseExternalStorageLedger = config.CaptiveCoreConfigUseExternalStorageLedger
+			ingestConfig.CaptiveCoreConfigUseDB = config.CaptiveCoreConfigUseDB
 		} else {
 			if config.StellarCoreDatabaseURL == "" {
 				return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go
index db2d13a0f9..19f979b84d 100644
--- a/services/horizon/internal/config.go
+++ b/services/horizon/internal/config.go
@@ -19,16 +19,16 @@ type Config struct {
 	Port               uint
 	AdminPort          uint
 
-	EnableCaptiveCoreIngestion                bool
-	UsingDefaultPubnetConfig                  bool
-	CaptiveCoreBinaryPath                     string
-	RemoteCaptiveCoreURL                      string
-	CaptiveCoreConfigPath                     string
-	CaptiveCoreTomlParams                     ledgerbackend.CaptiveCoreTomlParams
-	CaptiveCoreToml                           *ledgerbackend.CaptiveCoreToml
-	CaptiveCoreStoragePath                    string
-	CaptiveCoreReuseStoragePath               bool
-	CaptiveCoreConfigUseExternalStorageLedger bool
+	EnableCaptiveCoreIngestion  bool
+	UsingDefaultPubnetConfig    bool
+	CaptiveCoreBinaryPath       string
+	RemoteCaptiveCoreURL        string
+	CaptiveCoreConfigPath       string
+	CaptiveCoreTomlParams       ledgerbackend.CaptiveCoreTomlParams
+	CaptiveCoreToml             *ledgerbackend.CaptiveCoreToml
+	CaptiveCoreStoragePath      string
+	CaptiveCoreReuseStoragePath bool
+	CaptiveCoreConfigUseDB      bool
 
 	StellarCoreDatabaseURL string
 	StellarCoreURL         string
diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go
index d29b0ec522..13d7b9133a 100644
--- a/services/horizon/internal/flags.go
+++ b/services/horizon/internal/flags.go
@@ -36,8 +36,8 @@ const (
 	captiveCoreConfigAppendPathName = "captive-core-config-append-path"
 	// CaptiveCoreConfigPathName is the command line flag for configuring the path to the captive core configuration file
 	CaptiveCoreConfigPathName = "captive-core-config-path"
-	// captive-core-use-external-storage is the command line flag for enabling captive core runtime to use an external db url connection rather than RAM for ledger states
-	CaptiveCoreConfigUseExternalStorageLedger = "captive-core-use-external-storage"
+	// captive-core-use-db is the command line flag for enabling captive core runtime to use an external db url connection rather than RAM for ledger states
+	CaptiveCoreConfigUseDB = "captive-core-use-db"
 
 	captiveCoreMigrationHint = "If you are migrating from Horizon 1.x.y, start with the Migration Guide here: https://developers.stellar.org/docs/run-api-server/migrating/"
 )
@@ -173,7 +173,7 @@ func Flags() (*Config, support.ConfigOptions) {
 			},
 		},
 		&support.ConfigOption{
-			Name:        CaptiveCoreConfigUseExternalStorageLedger,
+			Name:        CaptiveCoreConfigUseDB,
 			OptType:     types.Bool,
 			FlagDefault: false,
 			Required:    false,
@@ -182,7 +182,7 @@ func Flags() (*Config, support.ConfigOptions) {
 						  Will result in several GB of space shifting out of RAM and to the external db persistence.\n
 						  The external db url is determined by the presence of DATABASE parameter in the captive-core-config-path or\n
 						  or if absent, the db will default to sqlite and the db file will be stored at location derived from captive-core-storage-path parameter.`,
-			ConfigKey: &config.CaptiveCoreConfigUseExternalStorageLedger,
+			ConfigKey: &config.CaptiveCoreConfigUseDB,
 		},
 		&support.ConfigOption{
 			Name:        "enable-captive-core-ingestion",
diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go
index 5f04082726..cf10deb129 100644
--- a/services/horizon/internal/ingest/main.go
+++ b/services/horizon/internal/ingest/main.go
@@ -74,15 +74,15 @@ type Config struct {
 	CaptiveCoreBinaryPath  string
 	CaptiveCoreStoragePath string
 	CaptiveCoreToml        *ledgerbackend.CaptiveCoreToml
+	CaptiveCoreConfigUseDB bool
 	RemoteCaptiveCoreURL   string
 	NetworkPassphrase      string
 
 	HistorySession    db.SessionInterface
 	HistoryArchiveURL string
 
-	DisableStateVerification                  bool
-	EnableExtendedLogLedgerStats              bool
-	CaptiveCoreConfigUseExternalStorageLedger bool
+	DisableStateVerification     bool
+	EnableExtendedLogLedgerStats bool
 
 	MaxReingestRetries          int
 	ReingestRetryBackoffSeconds int
@@ -223,15 +223,15 @@ func NewSystem(config Config) (System, error) {
 			logger := log.WithField("subservice", "stellar-core")
 			ledgerBackend, err = ledgerbackend.NewCaptive(
 				ledgerbackend.CaptiveCoreConfig{
-					BinaryPath:               config.CaptiveCoreBinaryPath,
-					StoragePath:              config.CaptiveCoreStoragePath,
-					UseExternalStorageLedger: config.CaptiveCoreConfigUseExternalStorageLedger,
-					Toml:                     config.CaptiveCoreToml,
-					NetworkPassphrase:        config.NetworkPassphrase,
-					HistoryArchiveURLs:       []string{config.HistoryArchiveURL},
-					CheckpointFrequency:      config.CheckpointFrequency,
-					Log:                      logger,
-					Context:                  ctx,
+					BinaryPath:          config.CaptiveCoreBinaryPath,
+					StoragePath:         config.CaptiveCoreStoragePath,
+					UseDB:               config.CaptiveCoreConfigUseDB,
+					Toml:                config.CaptiveCoreToml,
+					NetworkPassphrase:   config.NetworkPassphrase,
+					HistoryArchiveURLs:  []string{config.HistoryArchiveURL},
+					CheckpointFrequency: config.CheckpointFrequency,
+					Log:                 logger,
+					Context:             ctx,
 				},
 			)
 			if err != nil {
diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go
index c459dcedde..e2f92f225b 100644
--- a/services/horizon/internal/init.go
+++ b/services/horizon/internal/init.go
@@ -89,18 +89,18 @@ func initIngester(app *App) {
 		// TODO:
 		// Use the first archive for now. We don't have a mechanism to
 		// use multiple archives at the same time currently.
-		HistoryArchiveURL:                         app.config.HistoryArchiveURLs[0],
-		CheckpointFrequency:                       app.config.CheckpointFrequency,
-		StellarCoreURL:                            app.config.StellarCoreURL,
-		StellarCoreCursor:                         app.config.CursorName,
-		CaptiveCoreBinaryPath:                     app.config.CaptiveCoreBinaryPath,
-		CaptiveCoreStoragePath:                    app.config.CaptiveCoreStoragePath,
-		CaptiveCoreConfigUseExternalStorageLedger: app.config.CaptiveCoreConfigUseExternalStorageLedger,
-		CaptiveCoreToml:                           app.config.CaptiveCoreToml,
-		RemoteCaptiveCoreURL:                      app.config.RemoteCaptiveCoreURL,
-		EnableCaptiveCore:                         app.config.EnableCaptiveCoreIngestion,
-		DisableStateVerification:                  app.config.IngestDisableStateVerification,
-		EnableExtendedLogLedgerStats:              app.config.IngestEnableExtendedLogLedgerStats,
+		HistoryArchiveURL:            app.config.HistoryArchiveURLs[0],
+		CheckpointFrequency:          app.config.CheckpointFrequency,
+		StellarCoreURL:               app.config.StellarCoreURL,
+		StellarCoreCursor:            app.config.CursorName,
+		CaptiveCoreBinaryPath:        app.config.CaptiveCoreBinaryPath,
+		CaptiveCoreStoragePath:       app.config.CaptiveCoreStoragePath,
+		CaptiveCoreConfigUseDB:       app.config.CaptiveCoreConfigUseDB,
+		CaptiveCoreToml:              app.config.CaptiveCoreToml,
+		RemoteCaptiveCoreURL:         app.config.RemoteCaptiveCoreURL,
+		EnableCaptiveCore:            app.config.EnableCaptiveCoreIngestion,
+		DisableStateVerification:     app.config.IngestDisableStateVerification,
+		EnableExtendedLogLedgerStats: app.config.IngestEnableExtendedLogLedgerStats,
 	})
 
 	if err != nil {
diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go
index 82ca8c18b4..16ffa14d5c 100644
--- a/services/horizon/internal/integration/db_test.go
+++ b/services/horizon/internal/integration/db_test.go
@@ -240,8 +240,8 @@ func command(horizonConfig horizon.Config, args ...string) []string {
 		horizonConfig.CaptiveCoreBinaryPath,
 		"--captive-core-config-path",
 		horizonConfig.CaptiveCoreConfigPath,
-		"--captive-core-use-external-storage=" +
-			strconv.FormatBool(horizonConfig.CaptiveCoreConfigUseExternalStorageLedger),
+		"--captive-core-use-db=" +
+			strconv.FormatBool(horizonConfig.CaptiveCoreConfigUseDB),
 		"--enable-captive-core-ingestion=" + strconv.FormatBool(horizonConfig.EnableCaptiveCoreIngestion),
 		"--network-passphrase",
 		horizonConfig.NetworkPassphrase,
diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go
index a66fe421dd..622ec22f65 100644
--- a/services/horizon/internal/test/integration/integration.go
+++ b/services/horizon/internal/test/integration/integration.go
@@ -41,8 +41,8 @@ const (
 )
 
 var (
-	RunWithCaptiveCore              = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE") != ""
-	RunWithCaptiveCoreRemoteStorage = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_REMOTE_STORAGE") != ""
+	RunWithCaptiveCore      = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE") != ""
+	RunWithCaptiveCoreUseDB = os.Getenv("HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_USE_DB") != ""
 )
 
 type Config struct {
@@ -69,9 +69,9 @@ type Config struct {
 }
 
 type CaptiveConfig struct {
-	binaryPath       string
-	configPath       string
-	useRemoteStorage bool
+	binaryPath string
+	configPath string
+	useDB      bool
 }
 
 type Test struct {
@@ -155,8 +155,8 @@ func (i *Test) configureCaptiveCore() {
 		composePath := findDockerComposePath()
 		i.coreConfig.binaryPath = os.Getenv("CAPTIVE_CORE_BIN")
 		i.coreConfig.configPath = filepath.Join(composePath, "captive-core-integration-tests.cfg")
-		if RunWithCaptiveCoreRemoteStorage {
-			i.coreConfig.useRemoteStorage = true
+		if RunWithCaptiveCoreUseDB {
+			i.coreConfig.useDB = true
 		}
 	}
 
@@ -299,7 +299,7 @@ func (i *Test) StartHorizon() error {
 	hostname := "localhost"
 	coreBinaryPath := i.coreConfig.binaryPath
 	captiveCoreConfigPath := i.coreConfig.configPath
-	captiveCoreUseRemoteStorage := strconv.FormatBool(i.coreConfig.useRemoteStorage)
+	captiveCoreUseDB := strconv.FormatBool(i.coreConfig.useDB)
 
 	defaultArgs := map[string]string{
 		"stellar-core-url": i.coreClient.URL,
@@ -309,18 +309,18 @@ func (i *Test) StartHorizon() error {
 			hostname,
 			stellarCorePostgresPort,
 		),
-		"stellar-core-binary-path":          coreBinaryPath,
-		"captive-core-config-path":          captiveCoreConfigPath,
-		"captive-core-http-port":            "21626",
-		"captive-core-use-external-storage": captiveCoreUseRemoteStorage,
-		"enable-captive-core-ingestion":     strconv.FormatBool(len(coreBinaryPath) > 0),
-		"ingest":                            "true",
-		"history-archive-urls":              fmt.Sprintf("http://%s:%d", hostname, historyArchivePort),
-		"db-url":                            horizonPostgresURL,
-		"network-passphrase":                i.passPhrase,
-		"apply-migrations":                  "true",
-		"admin-port":                        strconv.Itoa(i.AdminPort()),
-		"port":                              "8000",
+		"stellar-core-binary-path":      coreBinaryPath,
+		"captive-core-config-path":      captiveCoreConfigPath,
+		"captive-core-http-port":        "21626",
+		"captive-core-use-db":           captiveCoreUseDB,
+		"enable-captive-core-ingestion": strconv.FormatBool(len(coreBinaryPath) > 0),
+		"ingest":                        "true",
+		"history-archive-urls":          fmt.Sprintf("http://%s:%d", hostname, historyArchivePort),
+		"db-url":                        horizonPostgresURL,
+		"network-passphrase":            i.passPhrase,
+		"apply-migrations":              "true",
+		"admin-port":                    strconv.Itoa(i.AdminPort()),
+		"port":                          "8000",
 		// due to ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING
 		"checkpoint-frequency": "8",
 		"per-hour-rate-limit":  "0", // disable rate limiting

From f68630142a19a151997f493235813e28cb07ee74 Mon Sep 17 00:00:00 2001
From: shawn <sreuland@users.noreply.github.com>
Date: Thu, 17 Feb 2022 12:29:57 -0800
Subject: [PATCH 17/17] #4038: incorporated UseDB in CaptiveCoreTomlParams and
 other PR feedback

---
 ingest/ledgerbackend/captive_core_backend.go  |  2 +-
 ingest/ledgerbackend/stellar_core_runner.go   | 29 +++++-------
 .../expected-offline-with-appendix-core.cfg   |  1 -
 .../expected-offline-with-extra-fields.cfg    |  1 -
 .../expected-offline-with-no-peer-port.cfg    |  1 -
 .../testdata/expected-online-core.cfg         |  1 -
 .../expected-online-with-no-http-port.cfg     |  1 -
 .../expected-online-with-no-peer-port.cfg     |  1 -
 ingest/ledgerbackend/toml.go                  |  4 +-
 ingest/ledgerbackend/toml_test.go             | 44 +++++++++++++++----
 services/horizon/internal/flags.go            | 10 +++++
 11 files changed, 62 insertions(+), 33 deletions(-)

diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
index fad024589d..1154096a8f 100644
--- a/ingest/ledgerbackend/captive_core_backend.go
+++ b/ingest/ledgerbackend/captive_core_backend.go
@@ -125,7 +125,7 @@ type CaptiveCoreConfig struct {
 	// it up entirely on shutdown.
 	StoragePath string
 
-	// UseExternalStorageLedger, when true, instructs the core invocation to use an external db url
+	// UseDB, when true, instructs the core invocation to use an external db url
 	// for ledger states rather than in memory(RAM). The external db url is determined by the presence
 	// of DATABASE parameter in the captive-core-config-path or if absent, the db will default to sqlite
 	// and the db file will be stored at location derived from StoragePath parameter.
diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go
index 0fbac765ce..b5b59b9b30 100644
--- a/ingest/ledgerbackend/stellar_core_runner.go
+++ b/ingest/ledgerbackend/stellar_core_runner.go
@@ -132,7 +132,7 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 		log: config.Log,
 	}
 
-	if conf, err := writeConf(config.Toml, mode, config, runner.getConfFileName()); err != nil {
+	if conf, err := writeConf(config.Toml, mode, runner.getConfFileName()); err != nil {
 		return nil, errors.Wrap(err, "error writing configuration")
 	} else {
 		runner.log.Debugf("captive core config file contents:\n%s", conf)
@@ -141,8 +141,8 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 	return runner, nil
 }
 
-func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig, location string) (string, error) {
-	text, err := generateConfig(captiveCoreToml, mode, config)
+func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, location string) (string, error) {
+	text, err := generateConfig(captiveCoreToml, mode)
 	if err != nil {
 		return "", err
 	}
@@ -150,9 +150,9 @@ func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, con
 	return string(text), ioutil.WriteFile(location, text, 0644)
 }
 
-func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, config CaptiveCoreConfig) ([]byte, error) {
-	var err error
+func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode) ([]byte, error) {
 	if mode == stellarCoreRunnerModeOffline {
+		var err error
 		captiveCoreToml, err = captiveCoreToml.CatchupToml()
 		if err != nil {
 			return nil, errors.Wrap(err, "could not generate catch up config")
@@ -263,7 +263,7 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 	}
 
 	rangeArg := fmt.Sprintf("%d/%d", to, to-from+1)
-	inMemory := "--in-memory"
+	params := []string{"catchup", rangeArg, "--metadata-output-stream", r.getPipeName()}
 
 	// horizon operator has specified to use external storage for captive core ledger state
 	// instruct captive core invocation to not use memory, and in that case
@@ -274,14 +274,11 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error {
 		if err := r.createCmd("new-db").Run(); err != nil {
 			return errors.Wrap(err, "error initializing core db")
 		}
-		inMemory = ""
+	} else {
+		params = append(params, "--in-memory")
 	}
 
-	r.cmd = r.createCmd(
-		"catchup", rangeArg,
-		"--metadata-output-stream", r.getPipeName(),
-		inMemory,
-	)
+	r.cmd = r.createCmd(params...)
 
 	var err error
 	r.pipe, err = r.start(r.cmd)
@@ -327,15 +324,13 @@ func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
 		// Do a quick catch-up to set the LCL in core to be our expected starting
 		// point.
 		if from > 2 {
-			// Can't catch up to the genesis ledger.
 			if err := r.createCmd("catchup", fmt.Sprintf("%d/0", from-1)).Run(); err != nil {
 				return errors.Wrap(err, "error runing stellar-core catchup")
 			}
-		} else {
-			if err := r.createCmd("catchup", "2/0").Run(); err != nil {
-				return errors.Wrap(err, "error runing stellar-core catchup")
-			}
+		} else if err := r.createCmd("catchup", "2/0").Run(); err != nil {
+			return errors.Wrap(err, "error runing stellar-core catchup")
 		}
+
 		r.cmd = r.createCmd(
 			"run",
 			"--metadata-output-stream",
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
index 178764beab..124abc435b 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
index a0f54c6f70..d7b41a0b81 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
index e5acfe73e6..9eca1ccad1 100644
--- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg
index 1fa85f8b84..57a5e7ff2c 100644
--- a/ingest/ledgerbackend/testdata/expected-online-core.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
index d164a8d67b..89e1762757 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
 LOG_FILE_PATH = ""
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
index 4ff5d3477a..1b65c5f318 100644
--- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
+++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
@@ -1,5 +1,4 @@
 # Generated file, do not edit
-DATABASE = "sqlite3://stellar.db"
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = "/var/stellar-core/test.log"
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
index 3935bee235..f6fedfa4a4 100644
--- a/ingest/ledgerbackend/toml.go
+++ b/ingest/ledgerbackend/toml.go
@@ -313,6 +313,8 @@ type CaptiveCoreTomlParams struct {
 	LogPath *string
 	// Strict is a flag which, if enabled, rejects Stellar Core toml fields which are not supported by captive core.
 	Strict bool
+	// If true, specifies that captive core should be invoked with on-disk rather than in-memory option for ledger state
+	UseDB bool
 }
 
 // NewCaptiveCoreTomlFromFile constructs a new CaptiveCoreToml instance by merging configuration
@@ -407,7 +409,7 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) {
 
 func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 
-	if !c.tree.Has("DATABASE") {
+	if params.UseDB && !c.tree.Has("DATABASE") {
 		c.Database = "sqlite3://stellar.db"
 	}
 
diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go
index 0465543a0c..35a52d67dd 100644
--- a/ingest/ledgerbackend/toml_test.go
+++ b/ingest/ledgerbackend/toml_test.go
@@ -225,6 +225,7 @@ func TestGenerateConfig(t *testing.T) {
 		httpPort     *uint
 		peerPort     *uint
 		logPath      *string
+		useDB        bool
 	}{
 		{
 			name:         "offline config with no appendix",
@@ -234,6 +235,7 @@ func TestGenerateConfig(t *testing.T) {
 			httpPort:     newUint(6789),
 			peerPort:     newUint(12345),
 			logPath:      nil,
+			useDB:        true,
 		},
 		{
 			name:         "offline config with no peer port",
@@ -309,6 +311,7 @@ func TestGenerateConfig(t *testing.T) {
 				PeerPort:           testCase.peerPort,
 				LogPath:            testCase.logPath,
 				Strict:             false,
+				UseDB:              testCase.useDB,
 			}
 			if testCase.appendPath != "" {
 				captiveCoreToml, err = NewCaptiveCoreTomlFromFile(testCase.appendPath, params)
@@ -317,7 +320,7 @@ func TestGenerateConfig(t *testing.T) {
 			}
 			assert.NoError(t, err)
 
-			configBytes, err := generateConfig(captiveCoreToml, testCase.mode, CaptiveCoreConfig{})
+			configBytes, err := generateConfig(captiveCoreToml, testCase.mode)
 			assert.NoError(t, err)
 
 			expectedByte, err := ioutil.ReadFile(testCase.expectedPath)
@@ -348,9 +351,7 @@ func TestExternalStorageConfigUsesDatabaseToml(t *testing.T) {
 	assert.NoError(t, err)
 	captiveCoreToml.Database = "sqlite3:///etc/defaults/stellar.db"
 
-	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
-		UseDB: true,
-	})
+	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline)
 
 	assert.NoError(t, err)
 	toml := CaptiveCoreToml{}
@@ -358,7 +359,7 @@ func TestExternalStorageConfigUsesDatabaseToml(t *testing.T) {
 	assert.Equal(t, toml.Database, "sqlite3:///etc/defaults/stellar.db")
 }
 
-func TestExternalStorageConfigDefaultsToSqlite(t *testing.T) {
+func TestDBConfigDefaultsToSqlite(t *testing.T) {
 	var err error
 	var captiveCoreToml *CaptiveCoreToml
 	httpPort := uint(8000)
@@ -372,17 +373,44 @@ func TestExternalStorageConfigDefaultsToSqlite(t *testing.T) {
 		PeerPort:           &peerPort,
 		LogPath:            &logPath,
 		Strict:             false,
+		UseDB:              true,
 	}
 
 	captiveCoreToml, err = NewCaptiveCoreToml(params)
 	assert.NoError(t, err)
 
-	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline, CaptiveCoreConfig{
-		UseDB: true,
-	})
+	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline)
 
 	assert.NoError(t, err)
 	toml := CaptiveCoreToml{}
 	toml.unmarshal(configBytes, true)
 	assert.Equal(t, toml.Database, "sqlite3://stellar.db")
 }
+
+func TestNonDBConfigDoesNotUpdateDatabase(t *testing.T) {
+	var err error
+	var captiveCoreToml *CaptiveCoreToml
+	httpPort := uint(8000)
+	peerPort := uint(8000)
+	logPath := "logPath"
+
+	// UseDB not set, which means it's false
+	params := CaptiveCoreTomlParams{
+		NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
+		HistoryArchiveURLs: []string{"http://localhost:1170"},
+		HTTPPort:           &httpPort,
+		PeerPort:           &peerPort,
+		LogPath:            &logPath,
+		Strict:             false,
+	}
+
+	captiveCoreToml, err = NewCaptiveCoreToml(params)
+	assert.NoError(t, err)
+
+	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOffline)
+
+	assert.NoError(t, err)
+	toml := CaptiveCoreToml{}
+	toml.unmarshal(configBytes, true)
+	assert.Equal(t, toml.Database, "")
+}
diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go
index 13d7b9133a..b774fd4189 100644
--- a/services/horizon/internal/flags.go
+++ b/services/horizon/internal/flags.go
@@ -182,6 +182,13 @@ func Flags() (*Config, support.ConfigOptions) {
 						  Will result in several GB of space shifting out of RAM and to the external db persistence.\n
 						  The external db url is determined by the presence of DATABASE parameter in the captive-core-config-path or\n
 						  or if absent, the db will default to sqlite and the db file will be stored at location derived from captive-core-storage-path parameter.`,
+			CustomSetValue: func(opt *support.ConfigOption) error {
+				if val := viper.GetBool(opt.Name); val {
+					config.CaptiveCoreConfigUseDB = val
+					config.CaptiveCoreTomlParams.UseDB = val
+				}
+				return nil
+			},
 			ConfigKey: &config.CaptiveCoreConfigUseDB,
 		},
 		&support.ConfigOption{
@@ -675,6 +682,9 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption
 		if config.StellarCoreDatabaseURL != "" {
 			return fmt.Errorf("Invalid config: --%s passed but --ingest not set. ", StellarCoreDBURLFlagName)
 		}
+		if config.CaptiveCoreConfigUseDB {
+			return fmt.Errorf("Invalid config: --%s has been set, but --ingest not set. ", CaptiveCoreConfigUseDB)
+		}
 	}
 
 	// Configure log file