From babac4f8a0b7c5afcd0ef5651dfd79e4faeced78 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Fri, 30 Jul 2021 17:16:25 +0200 Subject: [PATCH] services/horizon: Allow starting Horizon with minimal configuration (#3783) We want to make the time from Horizon installation to execution to be minimal. With less config values and reasonable defaults developers can start Horizon instance faster. This commit adds multiple changes to config parameters: * `--ingest` is set to `true` by default, * `--captive-core-config-path` is generated with default values based on the network passphrase. If the pubnet Horizon is started without custom config, the warning message will be printed in the log every hour. --- services/horizon/CHANGELOG.md | 1 + .../horizon/configs/captive-core-pubnet.cfg | 193 ++++++++++++++++++ .../horizon/configs/captive-core-testnet.cfg | 28 +++ services/horizon/internal/app.go | 17 ++ services/horizon/internal/config.go | 1 + services/horizon/internal/flags.go | 41 +++- .../scripts/build_release_artifacts/main.go | 5 + 7 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 services/horizon/configs/captive-core-pubnet.cfg create mode 100644 services/horizon/configs/captive-core-testnet.cfg diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 407f4e64dc..53cc8ab211 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -5,6 +5,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +* The `--ingest` flag is set by default. If `--captive-core-config-path` is not set, the config file is generated based on network passhprase. ([3783](https://github.com/stellar/go/pull/3783)) * Add a feature flag `--captive-core-reuse-storage-path`/`CAPTIVE_CORE_REUSE_STORAGE_PATH` that will reuse Captive Core's storage path for bucket files when applicable for better performance ([3750](https://github.com/stellar/go/pull/3750)). ## v2.6.1 diff --git a/services/horizon/configs/captive-core-pubnet.cfg b/services/horizon/configs/captive-core-pubnet.cfg new file mode 100644 index 0000000000..2fcbdf96b6 --- /dev/null +++ b/services/horizon/configs/captive-core-pubnet.cfg @@ -0,0 +1,193 @@ +# WARNING! Do not use this config in production. Quorum sets should +# be carefully selected manually. +NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" +HTTP_PORT=11626 + +[[HOME_DOMAINS]] +HOME_DOMAIN="stellar.org" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="satoshipay.io" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="lobstr.co" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="www.coinqvest.com" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="keybase.io" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="stellar.blockdaemon.com" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="wirexapp.com" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="sdf_1" +HOME_DOMAIN="stellar.org" +PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH" +ADDRESS="core-live-a.stellar.org:11625" +HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdf_2" +HOME_DOMAIN="stellar.org" +PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK" +ADDRESS="core-live-b.stellar.org:11625" +HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdf_3" +HOME_DOMAIN="stellar.org" +PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ" +ADDRESS="core-live-c.stellar.org:11625" +HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" + +[[VALIDATORS]] +NAME="satoshipay_singapore" +HOME_DOMAIN="satoshipay.io" +PUBLIC_KEY="GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT" +ADDRESS="stellar-sg-sin.satoshipay.io:11625" +HISTORY="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="satoshipay_iowa" +HOME_DOMAIN="satoshipay.io" +PUBLIC_KEY="GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY" +ADDRESS="stellar-us-iowa.satoshipay.io:11625" +HISTORY="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="satoshipay_frankfurt" +HOME_DOMAIN="satoshipay.io" +PUBLIC_KEY="GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE" +ADDRESS="stellar-de-fra.satoshipay.io:11625" +HISTORY="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_1_europe" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7" +ADDRESS="v1.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-1-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_2_europe" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GDXQB3OMMQ6MGG43PWFBZWBFKBBDUZIVSUDAZZTRAWQZKES2CDSE5HKJ" +ADDRESS="v2.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-2-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_3_north_america" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63" +ADDRESS="v3.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-3-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_4_asia" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J" +ADDRESS="v4.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-4-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_5_australia" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7" +ADDRESS="v5.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-5-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="coinqvest_hong_kong" +HOME_DOMAIN="www.coinqvest.com" +PUBLIC_KEY="GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z" +ADDRESS="hongkong.stellar.coinqvest.com:11625" +HISTORY="curl -sf https://hongkong.stellar.coinqvest.com/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="coinqvest_germany" +HOME_DOMAIN="www.coinqvest.com" +PUBLIC_KEY="GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN" +ADDRESS="germany.stellar.coinqvest.com:11625" +HISTORY="curl -sf https://germany.stellar.coinqvest.com/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="coinqvest_finland" +HOME_DOMAIN="www.coinqvest.com" +PUBLIC_KEY="GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T" +ADDRESS="finland.stellar.coinqvest.com:11625" +HISTORY="curl -sf https://finland.stellar.coinqvest.com/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="keybase_io" +HOME_DOMAIN="keybase.io" +PUBLIC_KEY="GCWJKM4EGTGJUVSWUJDPCQEOEP5LHSOFKSA4HALBTOO4T4H3HCHOM6UX" +ADDRESS="stellar0.keybase.io:11625" +HISTORY="curl -sf https://stellarhistory.keybase.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="keybase_1" +HOME_DOMAIN="keybase.io" +PUBLIC_KEY="GDKWELGJURRKXECG3HHFHXMRX64YWQPUHKCVRESOX3E5PM6DM4YXLZJM" +ADDRESS="stellar1.keybase.io:11625" +HISTORY="curl -sf https://stellarhistory1.keybase.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="keybase_2" +HOME_DOMAIN="keybase.io" +PUBLIC_KEY="GA35T3723UP2XJLC2H7MNL6VMKZZIFL2VW7XHMFFJKKIA2FJCYTLKFBW" +ADDRESS="stellar2.keybase.io:11625" +HISTORY="curl -sf https://stellarhistory2.keybase.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="Blockdaemon_Validator_1" +HOME_DOMAIN="stellar.blockdaemon.com" +PUBLIC_KEY="GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU" +ADDRESS="stellar-full-validator1.bdnodes.net" +HISTORY="curl -sf https://stellar-full-history1.bdnodes.net/{0} -o {1}" + +[[VALIDATORS]] +NAME="Blockdaemon_Validator_2" +HOME_DOMAIN="stellar.blockdaemon.com" +PUBLIC_KEY="GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C" +ADDRESS="stellar-full-validator2.bdnodes.net" +HISTORY="curl -sf https://stellar-full-history2.bdnodes.net/{0} -o {1}" + +[[VALIDATORS]] +NAME="Blockdaemon_Validator_3" +HOME_DOMAIN="stellar.blockdaemon.com" +PUBLIC_KEY="GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4" +ADDRESS="stellar-full-validator3.bdnodes.net" +HISTORY="curl -sf https://stellar-full-history3.bdnodes.net/{0} -o {1}" + +[[VALIDATORS]] +NAME="wirexUS" +ADDRESS="us.stellar.wirexapp.com" +HOME_DOMAIN="wirexapp.com" +PUBLIC_KEY="GDXUKFGG76WJC7ACEH3JUPLKM5N5S76QSMNDBONREUXPCZYVPOLFWXUS" +HISTORY="curl -sf http://wxhorizonusstga1.blob.core.windows.net/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="wirexUK" +ADDRESS="uk.stellar.wirexapp.com" +HOME_DOMAIN="wirexapp.com" +PUBLIC_KEY="GBBQQT3EIUSXRJC6TGUCGVA3FVPXVZLGG3OJYACWBEWYBHU46WJLWXEU" +HISTORY="curl -sf http://wxhorizonukstga1.blob.core.windows.net/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="wirexSG" +ADDRESS="sg.stellar.wirexapp.com" +HOME_DOMAIN="wirexapp.com" +PUBLIC_KEY="GAB3GZIE6XAYWXGZUDM4GMFFLJBFMLE2JDPUCWUZXMOMT3NHXDHEWXAS" +HISTORY="curl -sf http://wxhorizonasiastga1.blob.core.windows.net/history/{0} -o {1}" \ No newline at end of file diff --git a/services/horizon/configs/captive-core-testnet.cfg b/services/horizon/configs/captive-core-testnet.cfg new file mode 100644 index 0000000000..9abeecc8f5 --- /dev/null +++ b/services/horizon/configs/captive-core-testnet.cfg @@ -0,0 +1,28 @@ +NETWORK_PASSPHRASE="Test SDF Network ; September 2015" +UNSAFE_QUORUM=true +FAILURE_SAFETY=1 + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="core-testnet1.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdf_testnet_2" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP" +ADDRESS="core-testnet2.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdf_testnet_3" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z" +ADDRESS="core-testnet3.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}" \ No newline at end of file diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index b46cbfecd5..b84c8bc049 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -119,6 +119,23 @@ func (a *App) Serve() { // configure shutdown signal handler signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + if a.config.UsingDefaultPubnetConfig { + const warnMsg = "Horizon started using the default pubnet configuration. " + + "This is not safe! Please provide a custom --captive-core-config-path." + log.Warn(warnMsg) + go func() { + for { + select { + case <-time.After(time.Hour): + log.Warn(warnMsg) + case <-a.done: + return + } + } + }() + } + go func() { select { case <-signalChan: diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index c107413e31..aadd41408b 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -20,6 +20,7 @@ type Config struct { AdminPort uint EnableCaptiveCoreIngestion bool + UsingDefaultPubnetConfig bool CaptiveCoreBinaryPath string RemoteCaptiveCoreURL string CaptiveCoreConfigPath string diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index d19e7a1f97..0bc0036142 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -6,12 +6,14 @@ import ( stdLog "log" "os" "os/exec" + "path/filepath" "strings" "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/db2/schema" apkg "github.com/stellar/go/support/app" support "github.com/stellar/go/support/config" @@ -389,7 +391,7 @@ func Flags() (*Config, support.ConfigOptions) { Name: "ingest", ConfigKey: &config.Ingest, OptType: types.Bool, - FlagDefault: false, + FlagDefault: true, Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", }, &support.ConfigOption{ @@ -538,8 +540,41 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption if config.RemoteCaptiveCoreURL == "" && (binaryPath == "" || config.CaptiveCoreConfigPath == "") { if options.RequireCaptiveCoreConfig { - stdLog.Fatalf("Invalid config: captive core requires that both --%s and --%s are set. %s", - StellarCoreBinaryPathName, CaptiveCoreConfigPathName, captiveCoreMigrationHint) + var err error + errorMessage := fmt.Sprintf( + "Invalid config: captive core requires that both --%s and --%s are set. %s", + StellarCoreBinaryPathName, CaptiveCoreConfigPathName, captiveCoreMigrationHint, + ) + + var configFileName string + // Default config files will be located along the binary in the release archive. + switch config.NetworkPassphrase { + case network.TestNetworkPassphrase: + configFileName = "captive-core-testnet.cfg" + config.HistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-testnet/core_testnet_001/"} + case network.PublicNetworkPassphrase: + configFileName = "captive-core-pubnet.cfg" + config.HistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-live/core_live_001/"} + config.UsingDefaultPubnetConfig = true + default: + stdLog.Fatal(errorMessage) + } + + executablePath, err := os.Executable() + if err != nil { + stdLog.Fatal(errorMessage) + } + + config.CaptiveCoreConfigPath = filepath.Join(filepath.Dir(executablePath), configFileName) + if _, err = os.Stat(config.CaptiveCoreConfigPath); os.IsNotExist(err) { + stdLog.Fatal(errorMessage) + } + + config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase + config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromFile(config.CaptiveCoreConfigPath, config.CaptiveCoreTomlParams) + if err != nil { + stdLog.Fatalf("Invalid captive core toml file %v", err) + } } else { var err error config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs diff --git a/support/scripts/build_release_artifacts/main.go b/support/scripts/build_release_artifacts/main.go index 9fc08cec79..f0d4cdd9bb 100644 --- a/support/scripts/build_release_artifacts/main.go +++ b/support/scripts/build_release_artifacts/main.go @@ -305,6 +305,11 @@ func prepareDest(pkg, bin, version, os, arch string) string { run("cp", "COPYING", dest) run("cp", filepath.Join(pkg, "README.md"), dest) run("cp", filepath.Join(pkg, "CHANGELOG.md"), dest) + if bin == "horizon" { + // Add default config files for Captive-Core + run("cp", filepath.Join(pkg, "configs/captive-core-pubnet.cfg"), dest) + run("cp", filepath.Join(pkg, "configs/captive-core-testnet.cfg"), dest) + } return dest }