Merge pull request #5366 from bottlepay/postgres

kvdb: add postgres

.github/workflows/main.yml

@@ -206,7 +206,7 @@ jobs:
       matrix:
         unit_type:
           - btcd unit-cover
-          - unit tags=kvdb_etcd
+          - unit tags="kvdb_etcd kvdb_postgres"
           - travis-race
     steps:
       - name: git checkout

.travis.yml

@@ -61,6 +61,11 @@ jobs:
         - bash ./scripts/install_bitcoind.sh
         - make itest-parallel backend=bitcoind dbbackend=etcd
 
+    - name: Bitcoind Integration with postgres (txindex enabled)
+      script:
+        - bash ./scripts/install_bitcoind.sh
+        - make itest-parallel backend=bitcoind dbbackend=postgres
+
     - name: Bitcoind Integration (txindex disabled)
       script:
         - bash ./scripts/install_bitcoind.sh

Makefile

@@ -187,7 +187,20 @@ scratch: build
 
 check: unit itest
 
-itest-only:
+db-instance:
+ifeq ($(dbbackend),postgres)
+	# Remove a previous postgres instance if it exists.
+	docker rm lnd-postgres --force || echo "Starting new postgres container"
+
+	# Start a fresh postgres instance. Allow a maximum of 500 connections.
+	# This is required for the async benchmark to pass.
+	docker run --name lnd-postgres -e POSTGRES_PASSWORD=postgres -p 6432:5432 -d postgres -N 500
+
+	# Wait for the instance to be started.
+	sleep 3
+endif
+
+itest-only: db-instance
 	@$(call print, "Running integration tests with ${backend} backend.")
 	rm -rf lntest/itest/*.log lntest/itest/.logs-*; date
 	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS)
@@ -196,7 +209,7 @@ itest: build-itest itest-only
 
 itest-race: build-itest-race itest-only
 
-itest-parallel: build-itest
+itest-parallel: build-itest db-instance
 	@$(call print, "Running tests")
 	rm -rf lntest/itest/*.log lntest/itest/.logs-*; date
 	EXEC_SUFFIX=$(EXEC_SUFFIX) echo "$$(seq 0 $$(expr $(ITEST_PARALLELISM) - 1))" | xargs -P $(ITEST_PARALLELISM) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)

docs/postgres.md

@@ -0,0 +1,31 @@
+# Postgres support in LND
+
+With the introduction of the `kvdb` interface, LND can support multiple database
+backends. One of the supported backends is Postgres. This document
+describes how it can be configured.
+
+## Building LND with postgres support
+
+To build LND with postgres support, include the following build tag:
+
+```shell
+⛰  make tags="kvdb_postgres"
+```
+
+## Configuring Postgres for LND
+
+In order for LND to run on Postgres, an empty database should already exist. A
+database can be created via the usual ways (psql, pgadmin, etc). A user with
+access to this database is also required.
+
+Creation of a schema and the tables is handled by LND automatically.
+
+## Configuring LND for Postgres
+
+LND is configured for Postgres through the following configuration options:
+
+* `db.backend=postgres` to select the Postgres backend.
+* `db.postgres.dsn=...` to set the database connection string that includes
+  database, user and password.
+* `db.postgres.timeout=...` to set the connection timeout. If not set, no
+  timeout applies.

docs/release-notes/release-notes-0.14.0.md

@@ -38,6 +38,27 @@ instantaneous. Read the [guide on leader
 election](https://github.com/lightningnetwork/lnd/blob/master/docs/leader_election.md)
 for more information.
 
+### Postgres database support
+
+This release adds [support for Postgres as a database
+backend](https://github.com/lightningnetwork/lnd/pull/5366) to lnd. Postgres
+has several advantages over the default bbolt backend:
+* Better handling of large data sets.
+* On-the-fly database compaction (auto vacuum).
+* Database replication.
+* Inspect data while lnd is running (bbolt opens the database exclusively).
+* Usage of industry-standard tools to manage the stored data, get performance
+  metrics, etc.
+
+Furthermore, the SQL platform opens up possibilities to improve lnd's
+performance in the future. Bbolt's single-writer model is a severe performance
+bottleneck, whereas Postgres offers a variety of locking models. Additionally,
+structured tables reduce the need for custom serialization/deserialization code
+in `lnd`, saving developer time and limiting the potential for bugs.
+
+Instructions for enabling Postgres can be found in
+[docs/postgres.md](../postgres.md).
+
 ## Protocol Extensions
 
 ### Explicit Channel Negotiation

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/btcsuite/btcwallet/wallet/txrules v1.0.0
 	github.com/btcsuite/btcwallet/walletdb v1.3.6-0.20210803004036-eebed51155ec
 	github.com/btcsuite/btcwallet/wtxmgr v1.3.1-0.20210822222949-9b5a201c344c
-	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/davecgh/go-spew v1.1.1
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-errors/errors v1.0.1
@@ -24,6 +24,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.5.0
+	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jackpal/gateway v1.0.5
 	github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
@@ -55,7 +56,7 @@ require (
 	github.com/urfave/cli v1.20.0
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
+	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect