Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

services/horizon: Propogate *all* history archives through the ingestion interface. #4687

Merged
merged 3 commits into from
Nov 17, 2022
+22 −17
Merged

services/horizon: Propogate *all* history archives through the ingestion interface. #4687

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7ab4794
Propogate HA URLs through ingestion interface
Shaptic Nov 7, 2022
8780df8
Add changelog entry
Shaptic Nov 8, 2022
8af1bc4
Fixup after rebase w/ master
Shaptic Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions services/horizon/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@
All notable changes to this project will be documented in this
file. This project adheres to [Semantic Versioning](http://semver.org/).


## Unreleased

### Fixes

* The ingestion subsystem will now properly use a pool of history archives if more than one is provided ([#4687](https://github.com/stellar/go/pull/4687)).


## 2.22.1

**Upgrading to this version from <= v2.8.3 will trigger a state rebuild. During this process (which will take at least 10 minutes), Horizon will not ingest new ledgers.**
Expand Down
2 changes: 1 addition & 1 deletion services/horizon/cmd/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool,

ingestConfig := ingest.Config{
NetworkPassphrase: config.NetworkPassphrase,
HistoryArchiveURL: config.HistoryArchiveURLs[0],
HistoryArchiveURLs: config.HistoryArchiveURLs,
CheckpointFrequency: config.CheckpointFrequency,
ReingestEnabled: true,
MaxReingestRetries: int(retries),
Expand Down
8 changes: 4 additions & 4 deletions services/horizon/cmd/ingest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ var ingestVerifyRangeCmd = &cobra.Command{
ingestConfig := ingest.Config{
NetworkPassphrase: config.NetworkPassphrase,
HistorySession: horizonSession,
HistoryArchiveURL: config.HistoryArchiveURLs[0],
HistoryArchiveURLs: config.HistoryArchiveURLs,
Shaptic marked this conversation as resolved.
Show resolved Hide resolved
EnableCaptiveCore: config.EnableCaptiveCoreIngestion,
CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath,
CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB,
Expand Down Expand Up @@ -223,7 +223,7 @@ var ingestStressTestCmd = &cobra.Command{
ingestConfig := ingest.Config{
NetworkPassphrase: config.NetworkPassphrase,
HistorySession: horizonSession,
HistoryArchiveURL: config.HistoryArchiveURLs[0],
HistoryArchiveURLs: config.HistoryArchiveURLs,
EnableCaptiveCore: config.EnableCaptiveCoreIngestion,
RoundingSlippageFilter: config.RoundingSlippageFilter,
}
Expand Down Expand Up @@ -314,7 +314,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{
ingestConfig := ingest.Config{
NetworkPassphrase: config.NetworkPassphrase,
HistorySession: horizonSession,
HistoryArchiveURL: config.HistoryArchiveURLs[0],
HistoryArchiveURLs: config.HistoryArchiveURLs,
EnableCaptiveCore: config.EnableCaptiveCoreIngestion,
CheckpointFrequency: config.CheckpointFrequency,
RoundingSlippageFilter: config.RoundingSlippageFilter,
Expand Down Expand Up @@ -391,7 +391,7 @@ var ingestBuildStateCmd = &cobra.Command{
ingestConfig := ingest.Config{
NetworkPassphrase: config.NetworkPassphrase,
HistorySession: horizonSession,
HistoryArchiveURL: config.HistoryArchiveURLs[0],
HistoryArchiveURLs: config.HistoryArchiveURLs,
EnableCaptiveCore: config.EnableCaptiveCoreIngestion,
CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath,
CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB,
Expand Down
2 changes: 1 addition & 1 deletion services/horizon/internal/ingest/db_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func (s *DBTestSuite) SetupTest() {
sIface, err := NewSystem(Config{
CoreSession: s.tt.CoreSession(),
HistorySession: s.tt.HorizonSession(),
HistoryArchiveURL: "http://ignore.test",
HistoryArchiveURLs: []string{"http://ignore.test"},
DisableStateVerification: false,
CheckpointFrequency: 64,
})
Expand Down
10 changes: 5 additions & 5 deletions services/horizon/internal/ingest/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ type Config struct {
RemoteCaptiveCoreURL string
NetworkPassphrase string

HistorySession db.SessionInterface
HistoryArchiveURL string
HistorySession db.SessionInterface
HistoryArchiveURLs []string

DisableStateVerification bool
EnableReapLookupTables bool
Expand Down Expand Up @@ -214,8 +214,8 @@ type system struct {
func NewSystem(config Config) (System, error) {
ctx, cancel := context.WithCancel(context.Background())

archive, err := historyarchive.Connect(
config.HistoryArchiveURL,
archive, err := historyarchive.NewArchivePool(
config.HistoryArchiveURLs,
historyarchive.ConnectOptions{
Context: ctx,
NetworkPassphrase: config.NetworkPassphrase,
Expand Down Expand Up @@ -245,7 +245,7 @@ func NewSystem(config Config) (System, error) {
UseDB: config.CaptiveCoreConfigUseDB,
Toml: config.CaptiveCoreToml,
NetworkPassphrase: config.NetworkPassphrase,
HistoryArchiveURLs: []string{config.HistoryArchiveURL},
HistoryArchiveURLs: config.HistoryArchiveURLs,
CheckpointFrequency: config.CheckpointFrequency,
LedgerHashStore: ledgerbackend.NewHorizonDBLedgerHashStore(config.HistorySession),
Log: logger,
Expand Down
2 changes: 1 addition & 1 deletion services/horizon/internal/ingest/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func TestNewSystem(t *testing.T) {
CoreSession: &db.Session{DB: &sqlx.DB{}},
HistorySession: &db.Session{DB: &sqlx.DB{}},
DisableStateVerification: true,
HistoryArchiveURL: "https://history.stellar.org/prd/core-live/core_live_001",
HistoryArchiveURLs: []string{"https://history.stellar.org/prd/core-live/core_live_001"},
CheckpointFrequency: 64,
}

Expand Down
7 changes: 2 additions & 5 deletions services/horizon/internal/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,8 @@ func initIngester(app *App) {
HistorySession: mustNewDBSession(
db.IngestSubservice, app.config.DatabaseURL, ingest.MaxDBConnections, ingest.MaxDBConnections, app.prometheusRegistry,
),
NetworkPassphrase: app.config.NetworkPassphrase,
// TODO:
// Use the first archive for now. We don't have a mechanism to
// use multiple archives at the same time currently.
HistoryArchiveURL: app.config.HistoryArchiveURLs[0],
NetworkPassphrase: app.config.NetworkPassphrase,
HistoryArchiveURLs: app.config.HistoryArchiveURLs,
CheckpointFrequency: app.config.CheckpointFrequency,
StellarCoreURL: app.config.StellarCoreURL,
StellarCoreCursor: app.config.CursorName,
Expand Down