lncfg: add postgres

docs/postgres.md

@@ -0,0 +1,31 @@
+# Postgres support in LND
+
+With the introduction of the `kvdb` interface, LND can support multiple database
+backends. One of the supported backends is Postgres. This document
+describes how it can be configured.
+
+## Building LND with postgres support
+
+To build LND with postgres support, include the following build tag:
+
+```shell
+⛰  make tags="kvdb_postgres"
+```
+
+## Configuring Postgres for LND
+
+In order for LND to run on Postgres, an empty database should already exist. A
+database can be created via the usual ways (psql, pgadmin, etc). A user with
+access to this database is also required.
+
+Creation of a schema and the tables is handled by LND automatically.
+
+## Configuring LND for Postgres
+
+LND is configured for Postgres through the following configuration options:
+
+* `db.backend=postgres` to select the Postgres backend.
+* `db.postgres.dsn=...` to set the database connection string that includes
+  database, user and password.
+* `db.postgres.timeout=...` to set the connection timeout. If not set, no
+  timeout applies.

docs/release-notes/release-notes-0.14.0.md

@@ -38,6 +38,27 @@ instantaneous. Read the [guide on leader
 election](https://github.com/lightningnetwork/lnd/blob/master/docs/leader_election.md)
 for more information.
 
+### Postgres database support
+
+This release adds [support for Postgres as a database
+backend](https://github.com/lightningnetwork/lnd/pull/5366) to lnd. Postgres
+has several advantages over the default bbolt backend:
+* Better handling of large data sets.
+* On-the-fly database compaction (auto vacuum).
+* Database replication.
+* Inspect data while lnd is running (bbolt opens the database exclusively).
+* Usage of industry-standard tools to manage the stored data, get performance
+  metrics, etc.
+
+Furthermore, the SQL platform opens up possibilities to improve lnd's
+performance in the future. Bbolt's single-writer model is a severe performance
+bottleneck, whereas Postgres offers a variety of locking models. Additionally,
+structured tables reduce the need for custom serialization/deserialization code
+in `lnd`, saving developer time and limiting the potential for bugs.
+
+Instructions for enabling Postgres can be found in
+[docs/postgres.md](../postgres.md).
+
 ## Protocol Extensions
 
 ### Explicit Channel Negotiation

lncfg/db.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/lightningnetwork/lnd/kvdb"
 	"github.com/lightningnetwork/lnd/kvdb/etcd"
+	"github.com/lightningnetwork/lnd/kvdb/postgres"
 	"github.com/lightningnetwork/lnd/lnwallet/btcwallet"
 )
 
@@ -19,6 +20,7 @@ const (
 
 	BoltBackend                = "bolt"
 	EtcdBackend                = "etcd"
+	PostgresBackend            = "postgres"
 	DefaultBatchCommitInterval = 500 * time.Millisecond
 
 	// NSChannelDB is the namespace name that we use for the combined graph
@@ -53,6 +55,8 @@ type DB struct {
 	Etcd *etcd.Config `group:"etcd" namespace:"etcd" description:"Etcd settings."`
 
 	Bolt *kvdb.BoltConfig `group:"bolt" namespace:"bolt" description:"Bolt settings."`
+
+	Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."`
 }
 
 // DefaultDB creates and returns a new default DB config.
@@ -71,6 +75,10 @@ func DefaultDB() *DB {
 func (db *DB) Validate() error {
 	switch db.Backend {
 	case BoltBackend:
+	case PostgresBackend:
+		if db.Postgres.Dsn == "" {
+			return fmt.Errorf("postgres dsn must be set")
+		}
 
 	case EtcdBackend:
 		if !db.Etcd.Embedded && db.Etcd.Host == "" {
@@ -175,7 +183,8 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		}
 	}()
 
-	if db.Backend == EtcdBackend {
+	switch db.Backend {
+	case EtcdBackend:
 		// As long as the graph data, channel state and height hint
 		// cache are all still in the channel.db file in bolt, we
 		// replicate the same behavior here and use the same etcd
@@ -266,6 +275,88 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 			Remote:     true,
 			CloseFuncs: closeFuncs,
 		}, nil
+
+	case PostgresBackend:
+		postgresBackend, err := kvdb.Open(
+			kvdb.PostgresBackendName, ctx,
+			db.Postgres, NSChannelDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening postgres graph "+
+				"DB: %v", err)
+		}
+		closeFuncs[NSChannelDB] = postgresBackend.Close
+
+		postgresMacaroonBackend, err := kvdb.Open(
+			kvdb.PostgresBackendName, ctx,
+			db.Postgres, NSMacaroonDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening postgres "+
+				"macaroon DB: %v", err)
+		}
+		closeFuncs[NSMacaroonDB] = postgresMacaroonBackend.Close
+
+		postgresDecayedLogBackend, err := kvdb.Open(
+			kvdb.PostgresBackendName, ctx,
+			db.Postgres, NSDecayedLogDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening postgres "+
+				"decayed log DB: %v", err)
+		}
+		closeFuncs[NSDecayedLogDB] = postgresDecayedLogBackend.Close
+
+		postgresTowerClientBackend, err := kvdb.Open(
+			kvdb.PostgresBackendName, ctx,
+			db.Postgres, NSTowerClientDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening postgres tower "+
+				"client DB: %v", err)
+		}
+		closeFuncs[NSTowerClientDB] = postgresTowerClientBackend.Close
+
+		postgresTowerServerBackend, err := kvdb.Open(
+			kvdb.PostgresBackendName, ctx,
+			db.Postgres, NSTowerServerDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening postgres tower "+
+				"server DB: %v", err)
+		}
+		closeFuncs[NSTowerServerDB] = postgresTowerServerBackend.Close
+
+		postgresWalletBackend, err := kvdb.Open(
+			kvdb.PostgresBackendName, ctx,
+			db.Postgres, NSWalletDB,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error opening postgres macaroon "+
+				"DB: %v", err)
+		}
+		closeFuncs[NSWalletDB] = postgresWalletBackend.Close
+
+		returnEarly = false
+		return &DatabaseBackends{
+			GraphDB:       postgresBackend,
+			ChanStateDB:   postgresBackend,
+			HeightHintDB:  postgresBackend,
+			MacaroonDB:    postgresMacaroonBackend,
+			DecayedLogDB:  postgresDecayedLogBackend,
+			TowerClientDB: postgresTowerClientBackend,
+			TowerServerDB: postgresTowerServerBackend,
+			// The wallet loader will attempt to use/create the
+			// wallet in the replicated remote DB if we're running
+			// in a clustered environment. This will ensure that all
+			// members of the cluster have access to the same wallet
+			// state.
+			WalletDB: btcwallet.LoaderWithExternalWalletDB(
+				postgresWalletBackend,
+			),
+			Remote:     true,
+			CloseFuncs: closeFuncs,
+		}, nil
 	}
 
 	// We're using all bbolt based databases by default.

sample-lnd.conf

@@ -1135,6 +1135,13 @@ litecoin.node=ltcd
 ; testing with embedded etcd.
 ; db.etcd.embedded_log_file=/path/etcd.log
 
+[postgres]
+; Postgres connection string.
+; db.postgres.dsn=postgres://lnd:lnd@localhost:45432/lnd?sslmode=disable
+
+; Postgres connection timeout. Valid time units are {s, m, h}. Set to zero to
+; disable.
+; db.postgres.timeout=
 
 [cluster]