services/horizon: Allow starting Horizon with minimal configuration (#…

…3783) We want to make the time from Horizon installation to execution to be minimal. With less config values and reasonable defaults developers can start Horizon instance faster. This commit adds multiple changes to config parameters: * `--ingest` is set to `true` by default, * `--captive-core-config-path` is generated with default values based on the network passphrase. If the pubnet Horizon is started without custom config, the warning message will be printed in the log every hour.
stellar · Jul 30, 2021 · babac4f · babac4f
1 parent a1ff153
commit babac4f
Show file tree

Hide file tree

Showing 7 changed files with 283 additions and 3 deletions.
diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md
@@ -5,6 +5,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+* The `--ingest` flag is set by default. If `--captive-core-config-path` is not set, the config file is generated based on network passhprase. ([3783](https://github.com/stellar/go/pull/3783))
 * Add a feature flag `--captive-core-reuse-storage-path`/`CAPTIVE_CORE_REUSE_STORAGE_PATH` that will reuse Captive Core's storage path for bucket files when applicable for better performance ([3750](https://github.com/stellar/go/pull/3750)).
 
 ## v2.6.1

diff --git a/services/horizon/configs/captive-core-pubnet.cfg b/services/horizon/configs/captive-core-pubnet.cfg
@@ -0,0 +1,193 @@
+# WARNING! Do not use this config in production. Quorum sets should
+# be carefully selected manually.
+NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
+HTTP_PORT=11626
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="stellar.org"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="satoshipay.io"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="lobstr.co"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="www.coinqvest.com"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="keybase.io"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="stellar.blockdaemon.com"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="wirexapp.com"
+QUALITY="HIGH"
+
+[[VALIDATORS]]
+NAME="sdf_1"
+HOME_DOMAIN="stellar.org"
+PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH"
+ADDRESS="core-live-a.stellar.org:11625"
+HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_2"
+HOME_DOMAIN="stellar.org"
+PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK"
+ADDRESS="core-live-b.stellar.org:11625"
+HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_3"
+HOME_DOMAIN="stellar.org"
+PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ"
+ADDRESS="core-live-c.stellar.org:11625"
+HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="satoshipay_singapore"
+HOME_DOMAIN="satoshipay.io"
+PUBLIC_KEY="GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT"
+ADDRESS="stellar-sg-sin.satoshipay.io:11625"
+HISTORY="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="satoshipay_iowa"
+HOME_DOMAIN="satoshipay.io"
+PUBLIC_KEY="GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY"
+ADDRESS="stellar-us-iowa.satoshipay.io:11625"
+HISTORY="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="satoshipay_frankfurt"
+HOME_DOMAIN="satoshipay.io"
+PUBLIC_KEY="GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE"
+ADDRESS="stellar-de-fra.satoshipay.io:11625"
+HISTORY="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_1_europe"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7"
+ADDRESS="v1.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-1-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_2_europe"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GDXQB3OMMQ6MGG43PWFBZWBFKBBDUZIVSUDAZZTRAWQZKES2CDSE5HKJ"
+ADDRESS="v2.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-2-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_3_north_america"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63"
+ADDRESS="v3.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-3-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_4_asia"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J"
+ADDRESS="v4.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-4-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_5_australia"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7"
+ADDRESS="v5.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-5-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="coinqvest_hong_kong"
+HOME_DOMAIN="www.coinqvest.com"
+PUBLIC_KEY="GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z"
+ADDRESS="hongkong.stellar.coinqvest.com:11625"
+HISTORY="curl -sf https://hongkong.stellar.coinqvest.com/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="coinqvest_germany"
+HOME_DOMAIN="www.coinqvest.com"
+PUBLIC_KEY="GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN"
+ADDRESS="germany.stellar.coinqvest.com:11625"
+HISTORY="curl -sf https://germany.stellar.coinqvest.com/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="coinqvest_finland"
+HOME_DOMAIN="www.coinqvest.com"
+PUBLIC_KEY="GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T"
+ADDRESS="finland.stellar.coinqvest.com:11625"
+HISTORY="curl -sf https://finland.stellar.coinqvest.com/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="keybase_io"
+HOME_DOMAIN="keybase.io"
+PUBLIC_KEY="GCWJKM4EGTGJUVSWUJDPCQEOEP5LHSOFKSA4HALBTOO4T4H3HCHOM6UX"
+ADDRESS="stellar0.keybase.io:11625"
+HISTORY="curl -sf https://stellarhistory.keybase.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="keybase_1"
+HOME_DOMAIN="keybase.io"
+PUBLIC_KEY="GDKWELGJURRKXECG3HHFHXMRX64YWQPUHKCVRESOX3E5PM6DM4YXLZJM"
+ADDRESS="stellar1.keybase.io:11625"
+HISTORY="curl -sf https://stellarhistory1.keybase.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="keybase_2"
+HOME_DOMAIN="keybase.io"
+PUBLIC_KEY="GA35T3723UP2XJLC2H7MNL6VMKZZIFL2VW7XHMFFJKKIA2FJCYTLKFBW"
+ADDRESS="stellar2.keybase.io:11625"
+HISTORY="curl -sf https://stellarhistory2.keybase.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="Blockdaemon_Validator_1"
+HOME_DOMAIN="stellar.blockdaemon.com"
+PUBLIC_KEY="GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU"
+ADDRESS="stellar-full-validator1.bdnodes.net"
+HISTORY="curl -sf https://stellar-full-history1.bdnodes.net/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="Blockdaemon_Validator_2"
+HOME_DOMAIN="stellar.blockdaemon.com"
+PUBLIC_KEY="GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C"
+ADDRESS="stellar-full-validator2.bdnodes.net"
+HISTORY="curl -sf https://stellar-full-history2.bdnodes.net/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="Blockdaemon_Validator_3"
+HOME_DOMAIN="stellar.blockdaemon.com"
+PUBLIC_KEY="GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4"
+ADDRESS="stellar-full-validator3.bdnodes.net"
+HISTORY="curl -sf https://stellar-full-history3.bdnodes.net/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="wirexUS"
+ADDRESS="us.stellar.wirexapp.com"
+HOME_DOMAIN="wirexapp.com"
+PUBLIC_KEY="GDXUKFGG76WJC7ACEH3JUPLKM5N5S76QSMNDBONREUXPCZYVPOLFWXUS"
+HISTORY="curl -sf http://wxhorizonusstga1.blob.core.windows.net/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="wirexUK"
+ADDRESS="uk.stellar.wirexapp.com"
+HOME_DOMAIN="wirexapp.com"
+PUBLIC_KEY="GBBQQT3EIUSXRJC6TGUCGVA3FVPXVZLGG3OJYACWBEWYBHU46WJLWXEU"
+HISTORY="curl -sf http://wxhorizonukstga1.blob.core.windows.net/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="wirexSG"
+ADDRESS="sg.stellar.wirexapp.com"
+HOME_DOMAIN="wirexapp.com"
+PUBLIC_KEY="GAB3GZIE6XAYWXGZUDM4GMFFLJBFMLE2JDPUCWUZXMOMT3NHXDHEWXAS"
+HISTORY="curl -sf http://wxhorizonasiastga1.blob.core.windows.net/history/{0} -o {1}"
diff --git a/services/horizon/configs/captive-core-testnet.cfg b/services/horizon/configs/captive-core-testnet.cfg
@@ -0,0 +1,28 @@
+NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
+UNSAFE_QUORUM=true
+FAILURE_SAFETY=1
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="HIGH"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="core-testnet1.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_2"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP"
+ADDRESS="core-testnet2.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_3"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z"
+ADDRESS="core-testnet3.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}"
diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go
@@ -119,6 +119,23 @@ func (a *App) Serve() {
 	// configure shutdown signal handler
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+
+	if a.config.UsingDefaultPubnetConfig {
+		const warnMsg = "Horizon started using the default pubnet configuration. " +
+			"This is not safe! Please provide a custom --captive-core-config-path."
+		log.Warn(warnMsg)
+		go func() {
+			for {
+				select {
+				case <-time.After(time.Hour):
+					log.Warn(warnMsg)
+				case <-a.done:
+					return
+				}
+			}
+		}()
+	}
+
 	go func() {
 		select {
 		case <-signalChan:

diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go
@@ -20,6 +20,7 @@ type Config struct {
 	AdminPort          uint
 
 	EnableCaptiveCoreIngestion  bool
+	UsingDefaultPubnetConfig    bool
 	CaptiveCoreBinaryPath       string
 	RemoteCaptiveCoreURL        string
 	CaptiveCoreConfigPath       string

diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go
@@ -6,12 +6,14 @@ import (
 	stdLog "log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 
 	"github.com/stellar/go/ingest/ledgerbackend"
+	"github.com/stellar/go/network"
 	"github.com/stellar/go/services/horizon/internal/db2/schema"
 	apkg "github.com/stellar/go/support/app"
 	support "github.com/stellar/go/support/config"
@@ -389,7 +391,7 @@ func Flags() (*Config, support.ConfigOptions) {
 			Name:        "ingest",
 			ConfigKey:   &config.Ingest,
 			OptType:     types.Bool,
-			FlagDefault: false,
+			FlagDefault: true,
 			Usage:       "causes this horizon process to ingest data from stellar-core into horizon's db",
 		},
 		&support.ConfigOption{
@@ -538,8 +540,41 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption
 
 			if config.RemoteCaptiveCoreURL == "" && (binaryPath == "" || config.CaptiveCoreConfigPath == "") {
 				if options.RequireCaptiveCoreConfig {
-					stdLog.Fatalf("Invalid config: captive core requires that both --%s and --%s are set. %s",
-						StellarCoreBinaryPathName, CaptiveCoreConfigPathName, captiveCoreMigrationHint)
+					var err error
+					errorMessage := fmt.Sprintf(
+						"Invalid config: captive core requires that both --%s and --%s are set. %s",
+						StellarCoreBinaryPathName, CaptiveCoreConfigPathName, captiveCoreMigrationHint,
+					)
+
+					var configFileName string
+					// Default config files will be located along the binary in the release archive.
+					switch config.NetworkPassphrase {
+					case network.TestNetworkPassphrase:
+						configFileName = "captive-core-testnet.cfg"
+						config.HistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-testnet/core_testnet_001/"}
+					case network.PublicNetworkPassphrase:
+						configFileName = "captive-core-pubnet.cfg"
+						config.HistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-live/core_live_001/"}
+						config.UsingDefaultPubnetConfig = true
+					default:
+						stdLog.Fatal(errorMessage)
+					}
+
+					executablePath, err := os.Executable()
+					if err != nil {
+						stdLog.Fatal(errorMessage)
+					}
+
+					config.CaptiveCoreConfigPath = filepath.Join(filepath.Dir(executablePath), configFileName)
+					if _, err = os.Stat(config.CaptiveCoreConfigPath); os.IsNotExist(err) {
+						stdLog.Fatal(errorMessage)
+					}
+
+					config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase
+					config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromFile(config.CaptiveCoreConfigPath, config.CaptiveCoreTomlParams)
+					if err != nil {
+						stdLog.Fatalf("Invalid captive core toml file %v", err)
+					}
 				} else {
 					var err error
 					config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs

diff --git a/support/scripts/build_release_artifacts/main.go b/support/scripts/build_release_artifacts/main.go
@@ -305,6 +305,11 @@ func prepareDest(pkg, bin, version, os, arch string) string {
 	run("cp", "COPYING", dest)
 	run("cp", filepath.Join(pkg, "README.md"), dest)
 	run("cp", filepath.Join(pkg, "CHANGELOG.md"), dest)
+	if bin == "horizon" {
+		// Add default config files for Captive-Core
+		run("cp", filepath.Join(pkg, "configs/captive-core-pubnet.cfg"), dest)
+		run("cp", filepath.Join(pkg, "configs/captive-core-testnet.cfg"), dest)
+	}
 	return dest
 }