postgres: add itest

.travis.yml

@@ -61,6 +61,11 @@ jobs:
         - bash ./scripts/install_bitcoind.sh
         - make itest-parallel backend=bitcoind dbbackend=etcd
 
+    - name: Bitcoind Integration with postgres (txindex enabled)
+      script:
+        - bash ./scripts/install_bitcoind.sh
+        - make itest-parallel backend=bitcoind dbbackend=postgres
+
     - name: Bitcoind Integration (txindex disabled)
       script:
         - bash ./scripts/install_bitcoind.sh

go.mod

@@ -24,6 +24,8 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/grpc-ecosystem/grpc-gateway v1.14.3
+	github.com/jackc/pgx v3.6.2+incompatible
+	github.com/jackc/pgx/v4 v4.11.0
 	github.com/jackpal/gateway v1.0.5
 	github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad
 	github.com/jedib0t/go-pretty v4.3.0+incompatible

lntest/harness.go

@@ -86,6 +86,7 @@ type DatabaseBackend int
 const (
 	BackendBbolt DatabaseBackend = iota
 	BackendEtcd
+	BackendPostgres
 )
 
 // NewNetworkHarness creates a new network test harness.
@@ -1574,6 +1575,11 @@ func (n *NetworkHarness) BackupDb(hn *HarnessNode) error {
 		return errors.New("backup already created")
 	}
 
+	restart, err := n.SuspendNode(hn)
+	if err != nil {
+		return err
+	}
+
 	// Backup files.
 	tempDir, err := ioutil.TempDir("", "past-state")
 	if err != nil {
@@ -1586,6 +1592,22 @@ func (n *NetworkHarness) BackupDb(hn *HarnessNode) error {
 
 	hn.backupDbDir = tempDir
 
+	// Backup database.
+	if hn.postgresDbName != "" {
+		backupDbName := hn.postgresDbName + "_backup"
+		err := executePgQuery(
+			"CREATE DATABASE " + backupDbName + " WITH TEMPLATE " + hn.postgresDbName,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = restart()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -1605,5 +1627,22 @@ func (n *NetworkHarness) RestoreDb(hn *HarnessNode) error {
 	}
 	hn.backupDbDir = ""
 
+	// Restore database.
+	if hn.postgresDbName != "" {
+		backupDbName := hn.postgresDbName + "_backup"
+		err := executePgQuery(
+			"DROP DATABASE " + hn.postgresDbName,
+		)
+		if err != nil {
+			return err
+		}
+		err = executePgQuery(
+			"ALTER DATABASE " + backupDbName + " RENAME TO " + hn.postgresDbName,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

lntest/itest/lnd_test.go

@@ -81,7 +81,8 @@ var (
 	)
 
 	// dbBackendFlag specifies the backend to use
-	dbBackendFlag = flag.String("dbbackend", "bbolt", "Database backend (bbolt, etcd)")
+	dbBackendFlag = flag.String("dbbackend", "bbolt", "Database backend "+
+		"(bbolt, etcd, postgres)")
 )
 
 // getTestCaseSplitTranche returns the sub slice of the test cases that should
@@ -8281,6 +8282,13 @@ func testRevokedCloseRetribution(net *lntest.NetworkHarness, t *harnessTest) {
 		t.Fatalf("unable to copy database files: %v", err)
 	}
 
+	// Reconnect the peers after the restart that was needed for the db
+	// backup.
+	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
+	if err := net.ConnectNodes(ctxt, carol, net.Bob); err != nil {
+		t.Fatalf("unable to connect carol to bob: %v", err)
+	}
+
 	// Finally, send payments from Carol to Bob, consuming Bob's remaining
 	// payment hashes.
 	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
@@ -8527,6 +8535,13 @@ func testRevokedCloseRetributionZeroValueRemoteOutput(net *lntest.NetworkHarness
 		t.Fatalf("unable to copy database files: %v", err)
 	}
 
+	// Reconnect the peers after the restart that was needed for the db
+	// backup.
+	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
+	if err := net.ConnectNodes(ctxt, dave, carol); err != nil {
+		t.Fatalf("unable to connect dave to carol: %v", err)
+	}
+
 	// Finally, send payments from Dave to Carol, consuming Carol's remaining
 	// payment hashes.
 	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
@@ -9832,6 +9847,13 @@ func testDataLossProtection(net *lntest.NetworkHarness, t *harnessTest) {
 			t.Fatalf("unable to copy database files: %v", err)
 		}
 
+		// Reconnect the peers after the restart that was needed for the db
+		// backup.
+		ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
+		if err := net.ConnectNodes(ctxt, carol, node); err != nil {
+			t.Fatalf("unable to connect carol to node: %v", err)
+		}
+
 		// Finally, send more payments from , using the remaining
 		// payment hashes.
 		ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
@@ -14016,6 +14038,9 @@ func TestLightningNetworkDaemon(t *testing.T) {
 	case "etcd":
 		dbBackend = lntest.BackendEtcd
 
+	case "postgres":
+		dbBackend = lntest.BackendPostgres
+
 	default:
 		require.Fail(t, "unknown db backend")
 	}

lntest/itest/lnd_test_list_on_test.go

@@ -168,6 +168,8 @@ var allTestCases = []*testCase{
 		name: "switch offline delivery outgoing offline",
 		test: testSwitchOfflineDeliveryOutgoingOffline,
 	},
+	// TODO: Re-enable revoked close retribution tests when solution is in
+	// place for postgres db copy,.
 	{
 		// TODO(roasbeef): test always needs to be last as Bob's state
 		// is borked since we trick him into attempting to cheat Alice?

lntest/node.go

@@ -3,6 +3,7 @@ package lntest
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
@@ -23,6 +24,7 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 	"github.com/go-errors/errors"
+	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/lightningnetwork/lnd/chanbackup"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/invoicesrpc"
@@ -60,6 +62,8 @@ const (
 
 	// NeutrinoBackendName is the name of the neutrino backend.
 	NeutrinoBackendName = "neutrino"
+
+	postgresDsn = "postgres://postgres:postgres@localhost:6432/%s?sslmode=disable"
 )
 
 var (
@@ -91,6 +95,10 @@ var (
 	)
 )
 
+func postgresDatabaseDsn(dbName string) string {
+	return fmt.Sprintf(postgresDsn, dbName)
+}
+
 // NextAvailablePort returns the first port that is available for listening by
 // a new node. It panics if no port is found and the maximum available TCP port
 // is reached.
@@ -220,7 +228,8 @@ type NodeConfig struct {
 
 	FeeURL string
 
-	DbBackend DatabaseBackend
+	DbBackend   DatabaseBackend
+	PostgresDsn string
 }
 
 func (cfg NodeConfig) P2PAddr() string {
@@ -324,6 +333,10 @@ func (cfg NodeConfig) genArgs() []string {
 				NextAvailablePort(),
 			),
 		)
+
+	case BackendPostgres:
+		args = append(args, "--db.backend=postgres")
+		args = append(args, "--db.postgres.dsn="+cfg.PostgresDsn)
 	}
 
 	if cfg.FeeURL != "" {
@@ -393,6 +406,10 @@ type HarnessNode struct {
 
 	// backupDbDir is the path where a database backup is stored, if any.
 	backupDbDir string
+
+	// postgresDbName is the name of the postgres database where lnd data is
+	// stored in.
+	postgresDbName string
 }
 
 // Assert *HarnessNode implements the lnrpc.LightningClient interface.
@@ -428,6 +445,17 @@ func newNode(cfg NodeConfig) (*HarnessNode, error) {
 	// enabled.
 	cfg.AcceptKeySend = true
 
+	// Create temporary database.
+	var dbName string
+	if cfg.DbBackend == BackendPostgres {
+		var err error
+		dbName, err = createTempPgDb()
+		if err != nil {
+			return nil, err
+		}
+		cfg.PostgresDsn = postgresDatabaseDsn(dbName)
+	}
+
 	numActiveNodesMtx.Lock()
 	nodeNum := numActiveNodes
 	numActiveNodes++
@@ -442,9 +470,43 @@ func newNode(cfg NodeConfig) (*HarnessNode, error) {
 
 		closedChans:  make(map[wire.OutPoint]struct{}),
 		closeClients: make(map[wire.OutPoint][]chan struct{}),
+
+		postgresDbName: dbName,
 	}, nil
 }
 
+func createTempPgDb() (string, error) {
+	// Create random database name.
+	randBytes := make([]byte, 8)
+	_, err := rand.Read(randBytes)
+	if err != nil {
+		return "", err
+	}
+	dbName := "itest_" + hex.EncodeToString(randBytes)
+
+	// Create database.
+	err = executePgQuery("CREATE DATABASE " + dbName)
+	if err != nil {
+		return "", err
+	}
+
+	return dbName, nil
+}
+
+func executePgQuery(query string) error {
+	pool, err := pgxpool.Connect(
+		context.Background(),
+		postgresDatabaseDsn("postgres"),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to connect to database: %v", err)
+	}
+	defer pool.Close()
+
+	_, err = pool.Exec(context.Background(), query)
+	return err
+}
+
 // NewMiner creates a new miner using btcd backend. The logDir specifies the
 // miner node's log dir. When tests finished, during clean up, its logs are
 // copied to a file specified as logFilename.

scripts/itest_part.sh

@@ -11,6 +11,13 @@ NUM_TRANCHES=$2
 shift
 shift
 
+# Start postgres
+docker rm -f lnd-postgres
+docker run --name lnd-postgres -e POSTGRES_PASSWORD=postgres -p 6432:5432 -d postgres
+
+# Wait for it to be started
+sleep 3
+
 # Windows insists on having the .exe suffix for an executable, we need to add
 # that here if necessary.
 EXEC="$WORKDIR"/itest.test"$EXEC_SUFFIX"