Merge pull request #7252 from ellemouton/sqlite-pt2

multi: add sqlite backend option

.github/workflows/main.yml

@@ -158,6 +158,7 @@ jobs:
           - btcd unit-cover
           - unit tags="kvdb_etcd"
           - unit tags="kvdb_postgres"
+          - unit tags="kvdb_sqlite"
           - btcd unit-race
     steps:
       - name: git checkout
@@ -204,6 +205,8 @@ jobs:
             args: backend=bitcoind dbbackend=etcd
           - name: bitcoind-postgres
             args: backend=bitcoind dbbackend=postgres
+          - name: bitcoind-sqlite
+            args: backend=bitcoind dbbackend=sqlite
           - name: neutrino
             args: backend=neutrino
     steps:
@@ -287,6 +290,8 @@ jobs:
             args: backend=bitcoind dbbackend=etcd
           - name: bitcoind-postgres
             args: backend=bitcoind dbbackend=postgres
+          - name: bitcoind-sqlite
+            args: backend=bitcoind dbbackend=sqlite
           - name: neutrino
             args: backend=neutrino
     steps:
@@ -360,7 +365,7 @@ jobs:
       fail-fast: false
       matrix:
         pinned_dep:
-          - google.golang.org/grpc v1.38.0
+          - google.golang.org/grpc v1.41.0
           - github.com/golang/protobuf v1.5.2
 
     steps:

.github/workflows/release.yaml

@@ -94,7 +94,7 @@ jobs:
 
             Our release binaries are fully reproducible. Third parties are able to verify that the release binaries were produced properly without having to trust the release manager(s). See our [reproducible builds guide](https://github.com/lightningnetwork/lnd/tree/master/build/release) for how this can be achieved.
             The release binaries are compiled with `go${{ env.GO_VERSION }}`, which is required by verifiers to arrive at the same ones.
-            They include the following build tags: `autopilotrpc`, `signrpc`, `walletrpc`, `chainrpc`, `invoicesrpc`, `neutrinorpc`, `routerrpc`, `watchtowerrpc`, `monitoring`, `peersrpc`, `kvdb_postrgres`, and `kvdb_etcd`. Note that these are already included in the release script, so they do not need to be provided.
+            They include the following build tags: `autopilotrpc`, `signrpc`, `walletrpc`, `chainrpc`, `invoicesrpc`, `neutrinorpc`, `routerrpc`, `watchtowerrpc`, `monitoring`, `peersrpc`, `kvdb_postrgres`, `kvdb_etcd` and `kvdb_sqlite`. Note that these are already included in the release script, so they do not need to be provided.
 
             The `make release` command can be used to ensure one rebuilds with all the same flags used for the release. If one wishes to build for only a single platform, then `make release sys=<OS-ARCH> tag=<tag>` can be used. 
 

.golangci.yml

@@ -24,6 +24,7 @@ run:
     - watchtowerrpc
     - kvdb_etcd
     - kvdb_postgres
+    - kvdb_sqlite
 
 linters-settings:
   govet:

config.go

@@ -1405,12 +1405,18 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser,
 		)
 	}
 
+	towerDir := filepath.Join(
+		cfg.Watchtower.TowerDir,
+		cfg.registeredChains.PrimaryChain().String(),
+		lncfg.NormalizeNetwork(cfg.ActiveNetParams.Name),
+	)
+
 	// Create the lnd directory and all other sub-directories if they don't
 	// already exist. This makes sure that directory trees are also created
 	// for files that point to outside the lnddir.
 	dirs := []string{
 		lndDir, cfg.DataDir, cfg.networkDir,
-		cfg.LetsEncryptDir, cfg.Watchtower.TowerDir,
+		cfg.LetsEncryptDir, towerDir, cfg.graphDatabaseDir(),
 		filepath.Dir(cfg.TLSCertPath), filepath.Dir(cfg.TLSKeyPath),
 		filepath.Dir(cfg.AdminMacPath), filepath.Dir(cfg.ReadMacPath),
 		filepath.Dir(cfg.InvoiceMacPath),
@@ -1617,6 +1623,47 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser,
 		cfg.DB.Bolt.NoFreelistSync = !cfg.SyncFreelist
 	}
 
+	// Parse any extra sqlite pragma options that may have been provided
+	// to determine if they override any of the defaults that we will
+	// otherwise add.
+	var (
+		defaultSynchronous = true
+		defaultAutoVacuum  = true
+		defaultFullfsync   = true
+	)
+	for _, option := range cfg.DB.Sqlite.PragmaOptions {
+		switch {
+		case strings.HasPrefix(option, "synchronous="):
+			defaultSynchronous = false
+
+		case strings.HasPrefix(option, "auto_vacuum="):
+			defaultAutoVacuum = false
+
+		case strings.HasPrefix(option, "fullfsync="):
+			defaultFullfsync = false
+
+		default:
+		}
+	}
+
+	if defaultSynchronous {
+		cfg.DB.Sqlite.PragmaOptions = append(
+			cfg.DB.Sqlite.PragmaOptions, "synchronous=full",
+		)
+	}
+
+	if defaultAutoVacuum {
+		cfg.DB.Sqlite.PragmaOptions = append(
+			cfg.DB.Sqlite.PragmaOptions, "auto_vacuum=incremental",
+		)
+	}
+
+	if defaultFullfsync {
+		cfg.DB.Sqlite.PragmaOptions = append(
+			cfg.DB.Sqlite.PragmaOptions, "fullfsync=true",
+		)
+	}
+
 	// Ensure that the user hasn't chosen a remote-max-htlc value greater
 	// than the protocol maximum.
 	maxRemoteHtlcs := uint16(input.MaxHTLCNumber / 2)

config_builder.go

@@ -257,7 +257,7 @@ func (d *DefaultWalletImpl) BuildWalletConfig(ctx context.Context,
 	var neutrinoCS *neutrino.ChainService
 	if mainChain.Node == "neutrino" {
 		neutrinoBackend, neutrinoCleanUp, err := initNeutrinoBackend(
-			d.cfg, mainChain.ChainDir, blockCache,
+			ctx, d.cfg, mainChain.ChainDir, blockCache,
 		)
 		if err != nil {
 			err := fmt.Errorf("unable to initialize neutrino "+
@@ -830,7 +830,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 			cfg.Watchtower.TowerDir,
 			cfg.registeredChains.PrimaryChain().String(),
 			lncfg.NormalizeNetwork(cfg.ActiveNetParams.Name),
-		), cfg.WtClient.Active, cfg.Watchtower.Active,
+		), cfg.WtClient.Active, cfg.Watchtower.Active, d.logger,
 	)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to obtain database "+
@@ -1173,7 +1173,7 @@ func importWatchOnlyAccounts(wallet *wallet.Wallet,
 
 // initNeutrinoBackend inits a new instance of the neutrino light client
 // backend given a target chain directory to store the chain state.
-func initNeutrinoBackend(cfg *Config, chainDir string,
+func initNeutrinoBackend(ctx context.Context, cfg *Config, chainDir string,
 	blockCache *blockcache.BlockCache) (*neutrino.ChainService,
 	func(), error) {
 
@@ -1200,13 +1200,26 @@ func initNeutrinoBackend(cfg *Config, chainDir string,
 		return nil, nil, err
 	}
 
-	dbName := filepath.Join(dbPath, "neutrino.db")
-	db, err := walletdb.Create(
-		"bdb", dbName, !cfg.SyncFreelist, cfg.DB.Bolt.DBTimeout,
+	var (
+		db  walletdb.DB
+		err error
 	)
+	switch {
+	case cfg.DB.Backend == kvdb.SqliteBackendName:
+		db, err = kvdb.Open(
+			kvdb.SqliteBackendName, ctx, cfg.DB.Sqlite, dbPath,
+			lncfg.SqliteNeutrinoDBName, lncfg.NSNeutrinoDB,
+		)
+
+	default:
+		dbName := filepath.Join(dbPath, "neutrino.db")
+		db, err = walletdb.Create(
+			"bdb", dbName, !cfg.SyncFreelist, cfg.DB.Bolt.DBTimeout,
+		)
+	}
 	if err != nil {
-		return nil, nil, fmt.Errorf("unable to create neutrino "+
-			"database: %v", err)
+		return nil, nil, fmt.Errorf("unable to create "+
+			"neutrino database: %v", err)
 	}
 
 	headerStateAssertion, err := parseHeaderStateAssertion(

docs/INSTALL.md

@@ -254,6 +254,7 @@ will have the following tags:
 - [monitoring](/monitoring) (for Prometheus integration)
 - [peersrpc](/lnrpc/peersrpc/peers.proto)
 - [kvdb_postrgres](/docs/postgres.md)
+- [kvdb_sqlite](/docs/sqlite.md)
 - [kvdb_etcd](/docs/etcd.md)
 
 The `dev` tag is used for development builds, and is not included in the

docs/release-notes/release-notes-0.16.0.md

@@ -356,8 +356,11 @@ Keysend](https://github.com/lightningnetwork/lnd/pull/7334).
 ## DB
 
 * [Add a sqlite backend 
-  option](https://github.com/lightningnetwork/lnd/pull/7251) to the kvdb 
-  package.
+  option](https://github.com/lightningnetwork/lnd/pull/7251) to the kvdb
+  package, and add it as a [backend option to
+  LND](https://github.com/lightningnetwork/lnd/pull/7252). Note that with this
+  upgrade, support for the `dragonfly-amd64`, `netbsd-386`, `netbsd-arm64` and
+  `openbsd-386` platforms has been dropped.
 
 * [Bumped etcd dependencies to
   3.5.7](https://github.com/lightningnetwork/lnd/pull/7353) to resolve linking

docs/sqlite.md

@@ -0,0 +1,72 @@
+# SQLite support in LND
+
+With the introduction of the `kvdb` interface, LND can support multiple database
+backends. One of the supported backends is 
+[sqlite](https://www.sqlite.org/index.html). This document describes how use 
+LND with a sqlite backend.
+
+Note that for the time being, the sqlite backend option can only be set for 
+_new_ nodes. Setting the option for an existing node will not migrate the data. 
+
+## Supported platforms and architectures
+
+Note that the sqlite backend is _not_ supported for Windows (386/ARM) or for 
+Linux (PPC/MIPS) due to these platforms [not being supported by the sqlite 
+driver library.](
+https://pkg.go.dev/modernc.org/sqlite#hdr-Supported_platforms_and_architectures)
+
+## Configuring LND for SQLite
+
+LND is configured for SQLite through the following configuration options:
+
+* `db.backend=sqlite` to select the SQLite backend.
+* `db.sqlite.timeout=...` to set the connection timeout. If not set, no
+  timeout applies.
+* `db.sqlite.busytimeout=...` to set the maximum amount of time that a db call 
+  should wait if the db is currently locked.
+* `db.sqlite.pragmaoptions=...` to set a list of pragma options to be applied 
+  to the db connection. See the 
+  [sqlite documentation](https://www.sqlite.org/pragma.html) for more 
+  information on the available pragma options.
+
+## Default pragma options
+
+Currently, the following pragma options are always set:
+
+```
+foreign_keys=on
+journal_mode=wal
+busy_timeout=5000 // Overried with the db.sqlite.busytimeout option.
+```
+
+The following pragma options are set by default but can be overridden using the
+`db.sqlite.pragmaoptions` option:
+
+``` 
+synchronous=full
+auto_vacuum=incremental
+fullfsync=true // Only meaningful on a Mac. 
+```
+
+## Auto-compaction
+
+To activate auto-compaction on startup, the `incremental_vacuum` pragma option
+should be used:
+```
+// Use N to restrict the maximum number of pages to be removed from the 
+// freelist.
+db.sqlite.pragmaoptions=incremental_vacuum(N) 
+
+// Omit N if the entire freelist should be cleared.
+db.sqlite.pragmaoptions=incremental_vacuum
+```
+
+Example as follows:
+```
+[db]
+db.backend=sqlite
+db.sqlite.timeout=0
+db.sqlite.busytimeout=10s
+db.sqlite.pragmaoptions=temp_store=memory
+db.sqlite.pragmaoptions=incremental_vacuum
+```