kvdb: use docker based fixture

The embedded postgres was unreliable and prone to port collisions. We
use a docker based fixture that has proven to be reliable in other
projects.

kvdb/backend.go

@@ -257,10 +257,9 @@ func GetTestBackend(t testing.TB, path, name string) (Backend, func()) {
 		key := filepath.Join(path, name)
 		keyHash := sha256.Sum256([]byte(key))
 
-		f, err := NewPostgresFixture(t, "test_"+hex.EncodeToString(
+		f := NewPostgresFixture(t, "test_"+hex.EncodeToString(
 			keyHash[:]),
 		)
-		require.NoError(t, err)
 
 		return f.DB(), func() {
 			_ = f.DB().Close()

kvdb/go.mod

@@ -4,12 +4,13 @@ require (
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcwallet/walletdb v1.3.6-0.20210803004036-eebed51155ec
 	github.com/davecgh/go-spew v1.1.1
-	github.com/fergusstrange/embedded-postgres v1.25.0
 	github.com/google/btree v1.0.1
 	github.com/jackc/pgconn v1.14.0
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
 	github.com/jackc/pgx/v4 v4.18.1
+	github.com/lib/pq v1.10.4
 	github.com/lightningnetwork/lnd/healthcheck v1.0.0
+	github.com/ory/dockertest/v3 v3.10.0
 	github.com/stretchr/testify v1.8.2
 	go.etcd.io/bbolt v1.3.7
 	go.etcd.io/etcd/api/v3 v3.5.7
@@ -21,20 +22,30 @@ require (
 )
 
 require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/containerd/continuity v0.3.0 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/docker/cli v20.10.17+incompatible // indirect
+	github.com/docker/docker v20.10.7+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -45,13 +56,18 @@ require (
 	github.com/json-iterator/go v1.1.11 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
-	github.com/lib/pq v1.10.4 // indirect
 	github.com/lightningnetwork/lnd/ticker v1.0.0 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
@@ -63,7 +79,9 @@ require (
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
-	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	go.etcd.io/etcd/client/v2 v2.305.7 // indirect
 	go.etcd.io/etcd/pkg/v3 v3.5.7 // indirect

kvdb/kvdb_no_postgres.go

@@ -12,8 +12,9 @@ import (
 
 const PostgresBackend = false
 
-func NewPostgresFixture(_ testing.TB, _ string) (postgres.Fixture, error) {
-	return nil, errors.New("postgres backend not available")
+func NewPostgresFixture(t testing.TB, _ string) postgres.Fixture {
+	t.Fatalf("postgres backend not available")
+	return nil
 }
 
 func StartEmbeddedPostgres() (func() error, error) {

kvdb/kvdb_postgres.go

@@ -11,8 +11,8 @@ import (
 
 const PostgresBackend = true
 
-func NewPostgresFixture(t testing.TB, dbName string) (postgres.Fixture, error) {
-	return postgres.NewFixture(t, dbName)
+func NewPostgresFixture(t testing.TB, dbName string) postgres.Fixture {
+	return postgres.NewTestPgFixture(t, dbName)
 }
 
 func StartEmbeddedPostgres() (func() error, error) {

kvdb/postgres/db_test.go

@@ -3,6 +3,7 @@
 package postgres
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -18,8 +19,7 @@ func TestInterface(t *testing.T) {
 	require.NoError(t, err)
 	defer stop()
 
-	f, err := NewFixture(t, "")
-	require.NoError(t, err)
+	f := NewTestPgFixture(t, "")
 
 	// dbType is the database type name for this driver.
 	const dbType = "postgres"
@@ -38,9 +38,7 @@ func TestPanic(t *testing.T) {
 	require.NoError(t, err)
 	defer stop()
 
-	f, err := NewFixture(t, "")
-	require.NoError(t, err)
-
+	f := NewTestPgFixture(t, "")
 	err = f.Db.Update(func(tx walletdb.ReadWriteTx) error {
 		bucket, err := tx.CreateTopLevelBucket([]byte("test"))
 		require.NoError(t, err)
@@ -58,5 +56,9 @@ func TestPanic(t *testing.T) {
 		return nil
 	}, func() {})
 
-	require.Contains(t, err.Error(), "terminating connection")
+	hasErr := err != nil &&
+		strings.Contains(err.Error(), "unexpected EOF") ||
+		strings.Contains(err.Error(), "terminating connection")
+
+	require.True(t, hasErr)
 }

kvdb/postgres/fixture.go

@@ -8,86 +8,167 @@ import (
 	"database/sql"
 	"encoding/hex"
 	"fmt"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/btcsuite/btcwallet/walletdb"
-	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+	_ "github.com/lib/pq"
 	"github.com/lightningnetwork/lnd/kvdb/sqlbase"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/require"
 )
 
 const (
-	testDsnTemplate = "postgres://postgres:postgres@localhost:9876/%v?sslmode=disable"
+	testPgUser  = "test"
+	testPgPass  = "test"
+	testPgDb    = "test"
+	PostgresTag = "16"
+
+	testDsnTemplate = "postgres://test:test@%s:%d/%s?sslmode=disable"
 	prefix          = "test"
+
+	testMaxConnections = 200
 )
 
-func getTestDsn(dbName string) string {
-	return fmt.Sprintf(testDsnTemplate, dbName)
-}
+var (
+	// DefaultExpiry is the default expiry time for the test fixture. This
+	// means a single unit test is not allowed to run more than 3 hours
+	// (which is the same duration as the default test timeout).
+	DefaultExpiry = 180 * time.Minute
+)
 
-var testPostgres *embeddedpostgres.EmbeddedPostgres
+func getTestDsn(host string, port int, dbName string) string {
+	return fmt.Sprintf(testDsnTemplate, host, port, dbName)
+}
 
-const testMaxConnections = 200
+var testPostgres *testPgFixture
+
+// testPgFixture is a test fixture that starts a Postgres 11 instance in a
+// docker container.
+type testPgFixture struct {
+	db       *sql.DB
+	pool     *dockertest.Pool
+	resource *dockertest.Resource
+	host     string
+	port     int
+	Dsn      string
+}
 
 // StartEmbeddedPostgres starts an embedded postgres instance. This only needs
 // to be done once, because NewFixture will create random new databases on every
 // call. It returns a stop closure that stops the database if called.
 func StartEmbeddedPostgres() (func() error, error) {
 	sqlbase.Init(testMaxConnections)
 
-	postgres := embeddedpostgres.NewDatabase(
-		embeddedpostgres.DefaultConfig().
-			Port(9876).
-			StartParameters(
-				map[string]string{
-					"max_connections": fmt.Sprintf(
-						"%d", testMaxConnections,
-					),
-				},
-			),
-	)
+	// uses a sensible default on windows (tcp/http) and linux/osx (socket)
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		return nil, fmt.Errorf("error creating pool: %w", err)
+	}
 
-	err := postgres.Start()
+	// pulls an image, creates a container based on it and runs it
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "postgres",
+		Tag:        PostgresTag,
+		Env: []string{
+			fmt.Sprintf("POSTGRES_USER=%v", testPgUser),
+			fmt.Sprintf("POSTGRES_PASSWORD=%v", testPgPass),
+			fmt.Sprintf("POSTGRES_DB=%v", testPgDb),
+			"listen_addresses='*'",
+		},
+		Cmd: []string{"postgres", "-c", "max_connections=200"},
+	}, func(config *docker.HostConfig) {
+		// Set AutoRemove to true so that stopped container goes away
+		// by itself.
+		config.AutoRemove = true
+		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+	})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error creating resource: %w", err)
 	}
 
-	testPostgres = postgres
+	hostAndPort := resource.GetHostPort("5432/tcp")
+	parts := strings.Split(hostAndPort, ":")
+	host := parts[0]
+	port, err := strconv.ParseInt(parts[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing port: %w", err)
+	}
+
+	pgFixture := &testPgFixture{
+		host: host,
+		port: int(port),
+	}
+	databaseURL := getTestDsn(host, int(port), testPgDb)
 
-	return testPostgres.Stop, nil
+	// Tell docker to hard kill the container in "expiry" seconds.
+	err = resource.Expire(uint(DefaultExpiry.Seconds()))
+	if err != nil {
+		return nil, fmt.Errorf("error setting expiry: %w", err)
+	}
+
+	// Exponential backoff-retry, because the application in the container
+	// might not be ready to accept connections yet
+	pool.MaxWait = 120 * time.Second
+
+	var testDB *sql.DB
+	err = pool.Retry(func() error {
+		testDB, err = sql.Open("postgres", databaseURL)
+		if err != nil {
+			return err
+		}
+		return testDB.Ping()
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error connecting to docker: %w", err)
+	}
+
+	// Now fill remaining fields of the fixture.
+	pgFixture.db = testDB
+	pgFixture.pool = pool
+	pgFixture.resource = resource
+	pgFixture.Dsn = databaseURL
+
+	testPostgres = pgFixture
+
+	return pgFixture.tearDown, nil
+}
+
+// tearDown stops the underlying docker container.
+func (f *testPgFixture) tearDown() error {
+	return f.pool.Purge(f.resource)
 }
 
-// NewFixture returns a new postgres test database. The database name is
-// randomly generated.
-func NewFixture(t testing.TB, dbName string) (*fixture, error) {
+// NewTestPgFixture constructs a new TestPgFixture starting up a docker
+// container running Postgres 11. The started container will expire in after
+// the passed duration.
+func NewTestPgFixture(t testing.TB, dbName string) *fixture {
 	if dbName == "" {
 		// Create random database name.
 		randBytes := make([]byte, 8)
 		_, err := rand.Read(randBytes)
-		if err != nil {
-			return nil, err
-		}
+		require.NoError(t, err)
 
 		dbName = "test_" + hex.EncodeToString(randBytes)
 	}
 
 	// Create database if it doesn't exist yet.
-	dbConn, err := sql.Open("pgx", getTestDsn("postgres"))
-	if err != nil {
-		return nil, err
-	}
+	dbConn, err := sql.Open("pgx", testPostgres.Dsn)
+	require.NoError(t, err)
 	defer dbConn.Close()
 
 	_, err = dbConn.ExecContext(
 		context.Background(), "CREATE DATABASE "+dbName,
 	)
 	if err != nil && !strings.Contains(err.Error(), "already exists") {
-		return nil, err
+		t.Fatalf("unable to create database: %v", err)
 	}
 
-	// Open database
-	dsn := getTestDsn(dbName)
+	// Open new database.
+	dsn := getTestDsn(testPostgres.host, testPostgres.port, dbName)
 	db, err := newPostgresBackend(
 		context.Background(),
 		&Config{
@@ -96,14 +177,16 @@ func NewFixture(t testing.TB, dbName string) (*fixture, error) {
 		},
 		prefix,
 	)
-	if err != nil {
-		return nil, err
-	}
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		_ = db.Close()
+	})
 
 	return &fixture{
 		Dsn: dsn,
 		Db:  db,
-	}, nil
+	}
 }
 
 type fixture struct {
@@ -123,7 +206,8 @@ func (b *fixture) Dump() (map[string]interface{}, error) {
 	}
 
 	rows, err := dbConn.Query(
-		"SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname='public'",
+		"SELECT tablename FROM pg_catalog.pg_tables WHERE " +
+			"schemaname='public'",
 	)
 	if err != nil {
 		return nil, err
@@ -186,3 +270,7 @@ func (b *fixture) Dump() (map[string]interface{}, error) {
 
 	return result, nil
 }
+
+func init() {
+	sqlbase.Init(testMaxConnections)
+}