Merge pull request #762 from go-mysql-org/atercattus/allow-to-run-tes…

…ts-in-local-docker

Allow to run tests in local environment + fix tests for it

Makefile

@@ -11,9 +11,19 @@ build:
 test:
 	go test --race -timeout 2m ./...
 
+test-local:
+	docker run --rm -d --network=host --name go-mysql-server \
+		-e MYSQL_ALLOW_EMPTY_PASSWORD=true \
+		-e MYSQL_DATABASE=test \
+		-v $${PWD}/docker/resources/replication.cnf:/etc/mysql/conf.d/replication.cnf \
+		mysql:5.7
+	docker/resources/waitfor.sh 127.0.0.1 3306 \
+		&& go test -race -v -timeout 2m ./... -gocheck.v
+	docker stop go-mysql-server
+
 fmt:
 	golangci-lint run --fix
 
 clean:
 	go clean -i ./...
-	@rm -rf ./bin
+	@rm -rf ./bin

canal/canal_test.go

@@ -1,20 +1,19 @@
 package canal
 
 import (
-	"flag"
 	"fmt"
 	"testing"
 	"time"
 
-	"github.com/go-mysql-org/go-mysql/mysql"
-	"github.com/go-mysql-org/go-mysql/replication"
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb/parser"
 	"github.com/siddontang/go-log/log"
-)
 
-var testHost = flag.String("host", "127.0.0.1", "MySQL host")
+	"github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/go-mysql-org/go-mysql/replication"
+	"github.com/go-mysql-org/go-mysql/test_util"
+)
 
 func Test(t *testing.T) {
 	TestingT(t)
@@ -38,7 +37,7 @@ const (
 
 func (s *canalTestSuite) SetUpSuite(c *C) {
 	cfg := NewDefaultConfig()
-	cfg.Addr = fmt.Sprintf("%s:3306", *testHost)
+	cfg.Addr = fmt.Sprintf("%s:%s", *test_util.MysqlHost, *test_util.MysqlPort)
 	cfg.User = "root"
 	cfg.HeartbeatPeriod = 200 * time.Millisecond
 	cfg.ReadTimeout = 300 * time.Millisecond

client/client_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pingcap/errors"
 
 	"github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/go-mysql-org/go-mysql/test_util"
 	"github.com/go-mysql-org/go-mysql/test_util/test_keys"
 )
 
@@ -19,7 +20,7 @@ type clientTestSuite struct {
 
 func (s *clientTestSuite) SetUpSuite(c *C) {
 	var err error
-	addr := fmt.Sprintf("%s:%s", *testHost, s.port)
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, s.port)
 	s.c, err = Connect(addr, *testUser, *testPassword, "")
 	if err != nil {
 		c.Fatal(err)
@@ -117,7 +118,7 @@ func (s *clientTestSuite) TestConn_SetCapability(c *C) {
 func (s *clientTestSuite) TestConn_TLS_Verify(c *C) {
 	// Verify that the provided tls.Config is used when attempting to connect to mysql.
 	// An empty tls.Config will result in a connection error.
-	addr := fmt.Sprintf("%s:%s", *testHost, s.port)
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, s.port)
 	_, err := Connect(addr, *testUser, *testPassword, *testDB, func(c *Conn) {
 		c.UseSSL(false)
 	})
@@ -133,7 +134,7 @@ func (s *clientTestSuite) TestConn_TLS_Verify(c *C) {
 
 func (s *clientTestSuite) TestConn_TLS_Skip_Verify(c *C) {
 	// An empty tls.Config will result in a connection error but we can configure to skip it.
-	addr := fmt.Sprintf("%s:%s", *testHost, s.port)
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, s.port)
 	_, err := Connect(addr, *testUser, *testPassword, *testDB, func(c *Conn) {
 		c.UseSSL(true)
 	})
@@ -145,7 +146,7 @@ func (s *clientTestSuite) TestConn_TLS_Certificate(c *C) {
 	// And if server uses auto-generated certificates, it will be an error like:
 	// "x509: certificate is valid for MySQL_Server_8.0.12_Auto_Generated_Server_Certificate, not not-a-valid-name"
 	tlsConfig := NewClientTLSConfig(test_keys.CaPem, test_keys.CertPem, test_keys.KeyPem, false, "not-a-valid-name")
-	addr := fmt.Sprintf("%s:%s", *testHost, s.port)
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, s.port)
 	_, err := Connect(addr, *testUser, *testPassword, *testDB, func(c *Conn) {
 		c.SetTLSConfig(tlsConfig)
 	})

client/common_test.go

@@ -6,22 +6,22 @@ import (
 	"testing"
 
 	. "github.com/pingcap/check"
-)
 
-var testHost = flag.String("host", "127.0.0.1", "MySQL server host")
+	"github.com/go-mysql-org/go-mysql/test_util"
+)
 
-// We cover the whole range of MySQL server versions using docker-compose to bind them to different ports for testing.
-// MySQL is constantly updating auth plugin to make it secure:
-// starting from MySQL 8.0.4, a new auth plugin is introduced, causing plain password auth to fail with error:
-// ERROR 1251 (08004): Client does not support authentication protocol requested by server; consider upgrading MySQL client
-// Hint: use docker-compose to start corresponding MySQL docker containers and add the their ports here
-var testPort = flag.String("port", "3306", "MySQL server port") // choose one or more form 5561,5641,3306,5722,8003,8012,8013, e.g. '3306,5722,8003'
 var testUser = flag.String("user", "root", "MySQL user")
 var testPassword = flag.String("pass", "", "MySQL password")
 var testDB = flag.String("db", "test", "MySQL test database")
 
 func Test(t *testing.T) {
-	segs := strings.Split(*testPort, ",")
+	// We cover the whole range of MySQL server versions using docker-compose to bind them to different ports for testing.
+	// MySQL is constantly updating auth plugin to make it secure:
+	// starting from MySQL 8.0.4, a new auth plugin is introduced, causing plain password auth to fail with error:
+	// ERROR 1251 (08004): Client does not support authentication protocol requested by server; consider upgrading MySQL client
+	// Hint: use docker-compose to start corresponding MySQL docker containers and add their ports here
+
+	segs := strings.Split(*test_util.MysqlPort, ",")
 	for _, seg := range segs {
 		Suite(&clientTestSuite{port: seg})
 		Suite(&connTestSuite{port: seg})

client/conn_test.go

@@ -7,6 +7,7 @@ import (
 	. "github.com/pingcap/check"
 
 	"github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/go-mysql-org/go-mysql/test_util"
 )
 
 type connTestSuite struct {
@@ -16,7 +17,7 @@ type connTestSuite struct {
 
 func (s *connTestSuite) SetUpSuite(c *C) {
 	var err error
-	addr := fmt.Sprintf("%s:%s", *testHost, s.port)
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, s.port)
 	s.c, err = Connect(addr, *testUser, *testPassword, "", func(c *Conn) {
 		// required for the ExecuteMultiple test
 		c.SetCapability(mysql.CLIENT_MULTI_STATEMENTS)

docker/resources/replication.cnf

@@ -0,0 +1,7 @@
+[mysqld]
+server-id=1
+log-bin=mysql
+binlog-format=row
+gtid-mode=ON
+enforce_gtid_consistency=ON
+#log_error_verbosity=2

docker/resources/waitfor.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+host=$1
+port=$2
+
+echo "Waiting for mysql at $host:$port"
+while true; do
+  docker run --rm -it --network=host mysql:5.7 mysql -h$host -P$port -e "SELECT RAND()" >/dev/null
+  if [[ $? -eq 0 ]]; then
+    echo 'Connected'
+    break
+  fi
+
+  echo 'Still waiting...'
+  sleep 1
+done

driver/driver_test.go

@@ -9,13 +9,10 @@ import (
 
 	"github.com/jmoiron/sqlx"
 	. "github.com/pingcap/check"
-)
 
-// Use docker mysql to test, mysql is 3306
-var testHost = flag.String("host", "127.0.0.1", "MySQL master host")
+	"github.com/go-mysql-org/go-mysql/test_util"
+)
 
-// possible choices for different MySQL versions are: 5561,5641,3306,5722,8003,8012
-var testPort = flag.Int("port", 3306, "MySQL server port")
 var testUser = flag.String("user", "root", "MySQL user")
 var testPassword = flag.String("pass", "", "MySQL password")
 var testDB = flag.String("db", "test", "MySQL test database")
@@ -31,8 +28,8 @@ type testDriverSuite struct {
 var _ = Suite(&testDriverSuite{})
 
 func (s *testDriverSuite) SetUpSuite(c *C) {
-	addr := fmt.Sprintf("%s:%d", *testHost, *testPort)
-	dsn := fmt.Sprintf("%s:%s@%s?%s", *testUser, *testPassword, addr, *testDB)
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, *test_util.MysqlPort)
+	dsn := fmt.Sprintf("%s:%s@%s/%s", *testUser, *testPassword, addr, *testDB)
 
 	var err error
 	s.db, err = sqlx.Open("mysql", dsn)

dump/schema_test.go

@@ -6,8 +6,10 @@ import (
 	"io"
 	"os"
 
-	"github.com/go-mysql-org/go-mysql/client"
 	. "github.com/pingcap/check"
+
+	"github.com/go-mysql-org/go-mysql/client"
+	"github.com/go-mysql-org/go-mysql/test_util"
 )
 
 type schemaTestSuite struct {
@@ -18,11 +20,13 @@ type schemaTestSuite struct {
 var _ = Suite(&schemaTestSuite{})
 
 func (s *schemaTestSuite) SetUpSuite(c *C) {
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, *test_util.MysqlPort)
+
 	var err error
-	s.conn, err = client.Connect(fmt.Sprintf("%s:%d", *host, *port), "root", "", "")
+	s.conn, err = client.Connect(addr, "root", "", "")
 	c.Assert(err, IsNil)
 
-	s.d, err = NewDumper(*execution, fmt.Sprintf("%s:%d", *host, *port), "root", "")
+	s.d, err = NewDumper(*execution, addr, "root", "")
 	c.Assert(err, IsNil)
 	c.Assert(s.d, NotNil)
 

dump/setup_test.go

@@ -8,10 +8,6 @@ import (
 	. "github.com/pingcap/check"
 )
 
-// use docker mysql for test
-var host = flag.String("host", "127.0.0.1", "MySQL host")
-var port = flag.Int("port", 3306, "MySQL port")
-
 var execution = flag.String("exec", "mysqldump", "mysqldump execution path")
 
 func Test(t *testing.T) {

failover/failover_test.go

@@ -6,11 +6,10 @@ import (
 	"testing"
 
 	. "github.com/pingcap/check"
+
+	"github.com/go-mysql-org/go-mysql/test_util"
 )
 
-// We will use go-mysql docker to test
-// go-mysql docker will build mysql 1-3 instances
-var host = flag.String("host", "127.0.0.1", "go-mysql docker container address")
 var enable_failover_test = flag.Bool("test-failover", false, "enable test failover")
 
 func Test(t *testing.T) {
@@ -33,7 +32,7 @@ func (s *failoverTestSuite) SetUpSuite(c *C) {
 	s.s = make([]*Server, len(ports))
 
 	for i := 0; i < len(ports); i++ {
-		s.s[i] = NewServer(fmt.Sprintf("%s:%d", *host, ports[i]), User{"root", ""}, User{"root", ""})
+		s.s[i] = NewServer(fmt.Sprintf("%s:%d", *test_util.MysqlHost, ports[i]), User{"root", ""}, User{"root", ""})
 	}
 
 	var err error

mysql/mysql_test.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/pingcap/check"
+
+	_ "github.com/go-mysql-org/go-mysql/test_util" // Will register common flags
 )
 
 func Test(t *testing.T) {

replication/backup_test.go

@@ -3,7 +3,6 @@ package replication
 import (
 	"context"
 	"os"
-	"sync"
 	"time"
 
 	. "github.com/pingcap/check"
@@ -17,27 +16,20 @@ func (t *testSyncerSuite) TestStartBackupEndInGivenTime(c *C) {
 
 	t.testExecute(c, "RESET MASTER")
 
-	var wg sync.WaitGroup
-	wg.Add(1)
-	defer wg.Wait()
-
-	go func() {
-		defer wg.Done()
-
+	for times := 1; times <= 2; times++ {
 		t.testSync(c, nil)
-
 		t.testExecute(c, "FLUSH LOGS")
+	}
 
-		t.testSync(c, nil)
-	}()
+	binlogDir := "./var"
 
-	os.RemoveAll("./var")
+	os.RemoveAll(binlogDir)
 	timeout := 2 * time.Second
 
 	done := make(chan bool)
 
 	go func() {
-		err := t.b.StartBackup("./var", mysql.Position{Name: "", Pos: uint32(0)}, timeout)
+		err := t.b.StartBackup(binlogDir, mysql.Position{Name: "", Pos: uint32(0)}, timeout)
 		c.Assert(err, IsNil)
 		done <- true
 	}()

replication/replication_test.go

@@ -15,11 +15,9 @@ import (
 
 	"github.com/go-mysql-org/go-mysql/client"
 	"github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/go-mysql-org/go-mysql/test_util"
 )
 
-// Use docker mysql to test, mysql is 3306, mariadb is 3316
-var testHost = flag.String("host", "127.0.0.1", "MySQL master host")
-
 var testOutputLogs = flag.Bool("out", false, "output binlog event")
 
 func TestBinLogSyncer(t *testing.T) {
@@ -267,7 +265,7 @@ func (t *testSyncerSuite) setupTest(c *C, flavor string) {
 		t.c.Close()
 	}
 
-	t.c, err = client.Connect(fmt.Sprintf("%s:%d", *testHost, port), "root", "", "")
+	t.c, err = client.Connect(fmt.Sprintf("%s:%d", *test_util.MysqlHost, port), "root", "", "")
 	if err != nil {
 		c.Skip(err.Error())
 	}
@@ -285,7 +283,7 @@ func (t *testSyncerSuite) setupTest(c *C, flavor string) {
 	cfg := BinlogSyncerConfig{
 		ServerID:   100,
 		Flavor:     flavor,
-		Host:       *testHost,
+		Host:       *test_util.MysqlHost,
 		Port:       port,
 		User:       "root",
 		Password:   "",

schema/schema_test.go

@@ -10,10 +10,9 @@ import (
 
 	"github.com/go-mysql-org/go-mysql/client"
 	_ "github.com/go-mysql-org/go-mysql/driver"
+	"github.com/go-mysql-org/go-mysql/test_util"
 )
 
-// use docker mysql for test
-var host = flag.String("host", "127.0.0.1", "MySQL host")
 var schema = flag.String("schema", "test", "MySQL Database")
 var pwd = flag.String("pwd", "", "MySQL password")
 
@@ -29,11 +28,13 @@ type schemaTestSuite struct {
 var _ = Suite(&schemaTestSuite{})
 
 func (s *schemaTestSuite) SetUpSuite(c *C) {
+	addr := fmt.Sprintf("%s:%s", *test_util.MysqlHost, *test_util.MysqlPort)
+
 	var err error
-	s.conn, err = client.Connect(fmt.Sprintf("%s:%d", *host, 3306), "root", *pwd, *schema)
+	s.conn, err = client.Connect(addr, "root", *pwd, *schema)
 	c.Assert(err, IsNil)
 
-	s.sqlDB, err = sql.Open("mysql", fmt.Sprintf("root:%s@%s:3306", *pwd, *host))
+	s.sqlDB, err = sql.Open("mysql", fmt.Sprintf("root:%s@%s", *pwd, addr))
 	c.Assert(err, IsNil)
 }