testdb.Manager: try to recreate pool after cleanup error (#778)

In recent changes from #769 / fb6b3e9, I inadvertently tweaked the logic
of the testdb.Manager to throw away / destroy a resource if it
encountered cleanup errors. Destroying the resource has the side effect
of causing the manager to try to reallocate a new database with a new
number, potentially higher than the number of test databases we've
created.

Instead, try to just make a new pool on the same database, and only
destroy the resource if that fails.

internal/testdb/db_with_pool.go

@@ -40,12 +40,10 @@ func (db *DBWithPool) release() {
 		if errors.Is(err, puddle.ErrClosedPool) {
 			db.logger.Debug("DBWithPool: pool is closed, re-creating", "dbName", db.dbName)
 
-			newPgxPool, err := pgxpool.NewWithConfig(ctx, db.res.Value().config)
-			if err != nil {
+			if err := db.recreatePool(ctx); err != nil {
 				db.res.Destroy()
 				return
 			}
-			db.res.Value().pool = newPgxPool
 		} else {
 			// Log any other ping error but proceed with cleanup.
 			db.logger.Debug("DBWithPool: pool ping returned error", "dbName", db.dbName, "err", err)
@@ -56,7 +54,11 @@ func (db *DBWithPool) release() {
 		db.logger.Debug("DBWithPool: release calling cleanup", "dbName", db.dbName)
 		if err := db.manager.cleanup(ctx, db.res.Value().pool); err != nil {
 			db.logger.Error("testdb.DBWithPool: Error during release cleanup", "err", err)
-			db.res.Destroy()
+
+			if err := db.recreatePool(ctx); err != nil {
+				db.res.Destroy()
+				return
+			}
 		}
 		db.logger.Debug("DBWithPool: release done with cleanup", "dbName", db.dbName)
 	}
@@ -69,3 +71,18 @@ func (db *DBWithPool) release() {
 func (db *DBWithPool) Pool() *pgxpool.Pool {
 	return db.res.Value().pool
 }
+
+func (db *DBWithPool) recreatePool(ctx context.Context) error {
+	db.logger.Debug("DBWithPool: recreatePool called", "dbName", db.dbName)
+	db.Pool().Close()
+
+	newPgxPool, err := pgxpool.NewWithConfig(ctx, db.res.Value().config)
+	if err != nil {
+		db.res.Destroy()
+		db.logger.Error("DBWithPool: recreatePool failed", "dbName", db.dbName, "err", err)
+		return err
+	}
+	db.logger.Debug("DBWithPool: recreatePool succeeded", "dbName", db.dbName)
+	db.res.Value().pool = newPgxPool
+	return nil
+}

internal/testdb/manager.go

@@ -104,6 +104,7 @@ func (m *Manager) allocatePool(ctx context.Context) (*poolWithDBName, error) {
 	if m.cleanup != nil {
 		m.logger.Debug("DBManager: allocatePool calling cleanup", "dbName", dbName)
 		if err := m.cleanup(ctx, pgxp); err != nil {
+			m.logger.Error("DBManager: error during allocatePool cleanup", "error", err)
 			pgxp.Close()
 			return nil, fmt.Errorf("error during cleanup: %w", err)
 		}