Merge branch 'master' of github.com:pingcap/dm into cant-start-relay

Makefile

@@ -40,16 +40,6 @@ ARCH      := "$(shell uname -s)"
 LINUX     := "Linux"
 MAC       := "Darwin"
 
-ifeq ($(ARCH), $(LINUX))
-	LDFLAGS += -X "github.com/pingcap/dm/dm/worker.SampleConfigFile=$(shell cat dm/worker/dm-worker.toml | base64 -w 0)"
-	LDFLAGS += -X "github.com/pingcap/dm/dm/master.SampleConfigFile=$(shell cat dm/master/dm-master.toml | base64 -w 0)"
-	LDFLAGS += -X "github.com/pingcap/dm/dm/config.SampleConfigFile=$(shell cat dm/master/source.yaml | base64 -w 0)"
-else
-	LDFLAGS += -X "github.com/pingcap/dm/dm/worker.SampleConfigFile=$(shell cat dm/worker/dm-worker.toml | base64)"
-	LDFLAGS += -X "github.com/pingcap/dm/dm/master.SampleConfigFile=$(shell cat dm/master/dm-master.toml | base64)"
-	LDFLAGS += -X "github.com/pingcap/dm/dm/config.SampleConfigFile=$(shell cat dm/master/source.yaml | base64)"
-endif
-
 .PHONY: build tools_setup test unit_test dm_integration_test_build integration_test \
 	coverage check dm-worker dm-master chaos-case dmctl debug-tools
 

dm/config/source.yaml

@@ -0,0 +1,40 @@
+
+#server id of slave for binlog replication
+#each instance (master and slave) in replication group should have different server id
+server-id: 101
+
+#represents a MySQL/MariaDB instance or a replication group
+source-id: mysql-replica-01
+
+#flavor: mysql/mariadb
+flavor: mysql
+
+#enable relay log
+enable-relay: false
+# relay-binlog-name: ''
+# relay-binlog-gtid: ''
+# relay-dir: ./relay_log
+
+#enable gtid in relay log unit
+enable-gtid: false
+
+#charset of DSN of source mysql/mariadb instance
+# charset: ''
+
+from:
+  host: 127.0.0.1
+  user: root
+  password: Up8156jArvIPymkVC+5LxkAT6rek
+  port: 3306
+
+#relay log purge strategy
+#purge:
+#  interval: 3600
+#  expires: 24
+#  remain-space: 15
+
+#task status checker
+#checker:
+#  check-enable: true
+#  backoff-rollback: 5m
+#  backoff-max: 5m

dm/config/source_config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"database/sql"
+	_ "embed"
 	"encoding/json"
 	"math"
 	"math/rand"
@@ -33,9 +34,10 @@ const (
 
 var getAllServerIDFunc = utils.GetAllServerID
 
-// SampleConfigFile is sample config file of source
-// later we can read it from dm/master/source.yaml
-// and assign it to SampleConfigFile while we build dm-ctl.
+// SampleConfigFile is sample config file of source.
+// The embed source.yaml is a copy of dm/master/source.yaml, because embed
+// can only match regular files in the current directory and subdirectories.
+//go:embed source.yaml
 var SampleConfigFile string
 
 // PurgeConfig is the configuration for Purger.

dm/config/source_config_test.go

@@ -334,3 +334,9 @@ func (t *testConfig) TestAdjustCaseSensitive(c *C) {
 
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
+
+func (t *testConfig) TestEmbedSampleFile(c *C) {
+	data, err := os.ReadFile("./source.yaml")
+	c.Assert(err, IsNil)
+	c.Assert(SampleConfigFile, Equals, string(data))
+}

dm/ctl/master/operate_source.go

@@ -15,11 +15,9 @@ package master
 
 import (
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -66,17 +64,7 @@ func operateSourceFunc(cmd *cobra.Command, _ []string) error {
 	}
 
 	if printSampleConfig {
-		if strings.TrimSpace(config.SampleConfigFile) == "" {
-			fmt.Println("sample config file of source is empty")
-		} else {
-			var rawConfig []byte
-			rawConfig, err = base64.StdEncoding.DecodeString(config.SampleConfigFile)
-			if err != nil {
-				fmt.Println("base64 decode config error")
-			} else {
-				fmt.Println(string(rawConfig))
-			}
-		}
+		fmt.Println(config.SampleConfigFile)
 		return nil
 	}
 

dm/master/config.go

@@ -15,7 +15,7 @@ package master
 
 import (
 	"bytes"
-	"encoding/base64"
+	_ "embed"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -48,14 +48,9 @@ const (
 	quotaBackendBytesLowerBound    = 500 * 1024 * 1024      // 500MB
 )
 
-var (
-	// EnableZap enable the zap logger in embed etcd.
-	EnableZap = false
-	// SampleConfigFile is sample config file of dm-master
-	// later we can read it from dm/master/dm-master.toml
-	// and assign it to SampleConfigFile while we build dm-master.
-	SampleConfigFile string
-)
+// SampleConfigFile is sample config file of dm-master.
+//go:embed dm-master.toml
+var SampleConfigFile string
 
 // NewConfig creates a config for dm-master.
 func NewConfig() *Config {
@@ -175,16 +170,7 @@ func (c *Config) Parse(arguments []string) error {
 	}
 
 	if c.printSampleConfig {
-		if strings.TrimSpace(SampleConfigFile) == "" {
-			fmt.Println("sample config file of dm-master is empty")
-		} else {
-			rawConfig, err2 := base64.StdEncoding.DecodeString(SampleConfigFile)
-			if err2 != nil {
-				fmt.Println("base64 decode config error:", err2)
-			} else {
-				fmt.Println(string(rawConfig))
-			}
-		}
+		fmt.Println(SampleConfigFile)
 		return flag.ErrHelp
 	}
 

dm/master/config_test.go

@@ -14,7 +14,6 @@
 package master
 
 import (
-	"encoding/base64"
 	"flag"
 	"fmt"
 	"net/url"
@@ -43,39 +42,16 @@ func (t *testConfigSuite) SetUpSuite(c *check.C) {
 }
 
 func (t *testConfigSuite) TestPrintSampleConfig(c *check.C) {
-	var (
-		buf    []byte
-		err    error
-		encode string
-		out    string
-	)
-
-	defer func() {
-		SampleConfigFile = ""
-	}()
-
-	// test valid sample config
-	out = capturer.CaptureStdout(func() {
-		buf, err = os.ReadFile(defaultConfigFile)
-		c.Assert(err, check.IsNil)
-		encode = base64.StdEncoding.EncodeToString(buf)
-		SampleConfigFile = encode
+	buf, err := os.ReadFile(defaultConfigFile)
+	c.Assert(err, check.IsNil)
 
+	// test print sample config
+	out := capturer.CaptureStdout(func() {
 		cfg := NewConfig()
 		err = cfg.Parse([]string{"-print-sample-config"})
 		c.Assert(err, check.ErrorMatches, flag.ErrHelp.Error())
 	})
 	c.Assert(strings.TrimSpace(out), check.Equals, strings.TrimSpace(string(buf)))
-
-	// test invalid base64 encoded sample config
-	out = capturer.CaptureStdout(func() {
-		SampleConfigFile = "invalid base64 encode string"
-
-		cfg := NewConfig()
-		err = cfg.Parse([]string{"-print-sample-config"})
-		c.Assert(err, check.ErrorMatches, flag.ErrHelp.Error())
-	})
-	c.Assert(strings.TrimSpace(out), check.Matches, "base64 decode config error:.*")
 }
 
 func (t *testConfigSuite) TestConfig(c *check.C) {

dm/worker/config.go

@@ -15,7 +15,7 @@ package worker
 
 import (
 	"bytes"
-	"encoding/base64"
+	_ "embed"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -31,11 +31,11 @@ import (
 	"github.com/pingcap/dm/pkg/utils"
 )
 
-// SampleConfigFile is sample config file of dm-worker
-// later we can read it from dm/worker/dm-worker.toml
-// and assign it to SampleConfigFile while we build dm-worker.
+// SampleConfigFile is sample config file of dm-worker.
+//go:embed dm-worker.toml
+var SampleConfigFile string
+
 var (
-	SampleConfigFile         string
 	defaultKeepAliveTTL      = int64(60)      // 1 minute
 	defaultRelayKeepAliveTTL = int64(60 * 30) // 30 minutes
 )
@@ -147,16 +147,7 @@ func (c *Config) Parse(arguments []string) error {
 	}
 
 	if c.printSampleConfig {
-		if strings.TrimSpace(SampleConfigFile) == "" {
-			fmt.Println("sample config file of dm-worker is empty")
-		} else {
-			rawConfig, err2 := base64.StdEncoding.DecodeString(SampleConfigFile)
-			if err2 != nil {
-				fmt.Println("base64 decode config error:", err2)
-			} else {
-				fmt.Println(string(rawConfig))
-			}
-		}
+		fmt.Println(SampleConfigFile)
 		return flag.ErrHelp
 	}
 

dm/worker/config_test.go

@@ -14,6 +14,11 @@
 package worker
 
 import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/kami-zh/go-capturer"
 	"github.com/pingcap/check"
 
 	"github.com/pingcap/dm/pkg/terror"
@@ -46,3 +51,16 @@ func (t *testConfigSuite) TestAdjustAddr(c *check.C) {
 	c.Assert(cfg.adjust(), check.IsNil)
 	c.Assert(cfg.AdvertiseAddr, check.Equals, cfg.WorkerAddr)
 }
+
+func (t *testConfigSuite) TestPrintSampleConfig(c *check.C) {
+	buf, err := os.ReadFile(defaultConfigFile)
+	c.Assert(err, check.IsNil)
+
+	// test print sample config
+	out := capturer.CaptureStdout(func() {
+		cfg := NewConfig()
+		err = cfg.Parse([]string{"-print-sample-config"})
+		c.Assert(err, check.ErrorMatches, flag.ErrHelp.Error())
+	})
+	c.Assert(strings.TrimSpace(out), check.Equals, strings.TrimSpace(string(buf)))
+}