dm/openapi: support config experimental feature (#3266) (#3285)

dm/dm/master/config.go

@@ -90,6 +90,10 @@ func NewConfig() *Config {
 	return cfg
 }
 
+type ExperimentalFeatures struct {
+	OpenAPI bool `toml:"openapi"`
+}
+
 // Config is the configuration for dm-master.
 type Config struct {
 	flagSet *flag.FlagSet
@@ -134,6 +138,8 @@ type Config struct {
 
 	printVersion      bool
 	printSampleConfig bool
+
+	ExperimentalFeatures ExperimentalFeatures `toml:"experimental"`
 }
 
 func (c *Config) String() string {

dm/dm/master/config_test.go

@@ -114,6 +114,7 @@ func (t *testConfigSuite) TestConfig(c *check.C) {
 			c.Assert(cfg.InitialClusterState, check.Equals, embed.ClusterStateFlagNew)
 			c.Assert(cfg.Join, check.Equals, "")
 			c.Assert(cfg.String(), check.Matches, fmt.Sprintf("{.*master-addr\":\"%s\".*}", masterAddr))
+			c.Assert(cfg.ExperimentalFeatures.OpenAPI, check.Equals, false)
 		}
 	}
 }

dm/dm/master/dm-master.toml

@@ -1,8 +1,8 @@
 # Master Configuration.
 
 # log configuration
-log-level = "info"
 log-file = "dm-master.log"
+log-level = "info"
 
 # dm-master listen address
 master-addr = ":8261"
@@ -39,5 +39,9 @@ rpc-timeout = "30s"
 # refilled at rate `rpc-rate-limit` tokens per second. Note `rpc-rate-limit`
 # is float64 type, so remember to add a decimal point and one trailing 0 if its
 # literal value happens to be an integer.
-rpc-rate-limit = 10.0
 rpc-rate-burst = 40
+rpc-rate-limit = 10.0
+
+# some experimental features
+[experimental]
+openapi = false

dm/dm/master/openapi_test.go

@@ -88,6 +88,7 @@ func (t *openAPISuite) TestRedirectRequestToLeader(c *check.C) {
 	cfg1.PeerUrls = tempurl.Alloc()
 	cfg1.AdvertisePeerUrls = cfg1.PeerUrls
 	cfg1.InitialCluster = fmt.Sprintf("%s=%s", cfg1.Name, cfg1.AdvertisePeerUrls)
+	cfg1.ExperimentalFeatures.OpenAPI = true
 
 	s1 := NewServer(cfg1)
 	c.Assert(s1.Start(ctx), check.IsNil)
@@ -107,6 +108,7 @@ func (t *openAPISuite) TestRedirectRequestToLeader(c *check.C) {
 	cfg2.PeerUrls = tempurl.Alloc()
 	cfg2.AdvertisePeerUrls = cfg2.PeerUrls
 	cfg2.Join = cfg1.MasterAddr // join to an existing cluster
+	cfg2.ExperimentalFeatures.OpenAPI = true
 
 	s2 := NewServer(cfg2)
 	c.Assert(s2.Start(ctx), check.IsNil)
@@ -129,6 +131,29 @@ func (t *openAPISuite) TestRedirectRequestToLeader(c *check.C) {
 	cancel()
 }
 
+func (t *openAPISuite) TestOpenAPIWillNotStartInDefaultConfig(c *check.C) {
+	// create a new cluster
+	cfg1 := NewConfig()
+	c.Assert(cfg1.Parse([]string{"-config=./dm-master.toml"}), check.IsNil)
+	cfg1.Name = "dm-master-1"
+	cfg1.DataDir = c.MkDir()
+	cfg1.MasterAddr = tempurl.Alloc()[len("http://"):]
+	cfg1.PeerUrls = tempurl.Alloc()
+	cfg1.AdvertisePeerUrls = cfg1.PeerUrls
+	cfg1.InitialCluster = fmt.Sprintf("%s=%s", cfg1.Name, cfg1.AdvertisePeerUrls)
+
+	s1 := NewServer(cfg1)
+	ctx, cancel := context.WithCancel(context.Background())
+	c.Assert(s1.Start(ctx), check.IsNil)
+	// wait the first one become the leader
+	c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool {
+		return s1.election.IsLeader() && s1.scheduler.Started()
+	}), check.IsTrue)
+	c.Assert(s1.echo, check.IsNil)
+	defer s1.Close()
+	cancel()
+}
+
 func (t *openAPISuite) TestSourceAPI(c *check.C) {
 	ctx, cancel := context.WithCancel(context.Background())
 	s := setupServer(ctx, c)
@@ -600,6 +625,7 @@ func setupServer(ctx context.Context, c *check.C) *Server {
 	cfg1.AdvertisePeerUrls = cfg1.PeerUrls
 	cfg1.AdvertiseAddr = cfg1.MasterAddr
 	cfg1.InitialCluster = fmt.Sprintf("%s=%s", cfg1.Name, cfg1.AdvertisePeerUrls)
+	cfg1.ExperimentalFeatures.OpenAPI = true
 
 	s1 := NewServer(cfg1)
 	c.Assert(s1.Start(ctx), check.IsNil)

dm/dm/master/server.go

@@ -184,15 +184,17 @@ func (s *Server) Start(ctx context.Context) (err error) {
 	// And curl or safari browser does trigger this problem.
 	// But I haven't figured it out.
 	// (maybe more requests are sent from chrome or its extensions).
-	initOpenAPIErr := s.InitOpenAPIHandles()
-	if initOpenAPIErr != nil {
-		return terror.ErrOpenAPICommonError.Delegate(initOpenAPIErr)
-	}
+
 	userHandles := map[string]http.Handler{
-		"/apis/":   apiHandler,
-		"/status":  getStatusHandle(),
-		"/debug/":  getDebugHandler(),
-		"/api/v1/": s.echo,
+		"/apis/":  apiHandler,
+		"/status": getStatusHandle(),
+		"/debug/": getDebugHandler(),
+	}
+	if s.cfg.ExperimentalFeatures.OpenAPI {
+		if initOpenAPIErr := s.InitOpenAPIHandles(); initOpenAPIErr != nil {
+			return terror.ErrOpenAPICommonError.Delegate(initOpenAPIErr)
+		}
+		userHandles["/api/v1/"] = s.echo
 	}
 
 	// gRPC API server

dm/tests/dmctl_basic/conf/get_master1.toml

@@ -24,3 +24,6 @@ v1-sources-path = ""
 ssl-ca = ""
 ssl-cert = ""
 ssl-key = ""
+
+[experimental]
+  openapi = false

dm/tests/openapi/client/openapi_task_check

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 import sys
 import requests
 
@@ -158,6 +159,20 @@ def get_task_status_failed(task_name):
     assert resp.status_code == 400
 
 
+def get_illegal_char_task_status_failed():
+    # task name contains illegal char but api server can handle it.
+    # return 400 is because of the task is not started.
+    url = API_ENDPOINT + "/" + ILLEGAL_CHAR_TASK_NAME + "/status"
+    resp = requests.get(url=url)
+    print("get_illegal_char_task_status_failed resp=", resp.json())
+    assert resp.status_code == 400
+    if sys.version_info.major == 2:
+        # need decode in python2
+        assert ILLEGAL_CHAR_TASK_NAME.decode("utf-8") in resp.json()["error_msg"]
+    else:
+        assert ILLEGAL_CHAR_TASK_NAME in resp.json()["error_msg"]
+
+
 def get_task_status_success(task_name, total):
     url = API_ENDPOINT + "/" + task_name + "/status"
     resp = requests.get(url=url)

dm/tests/openapi/conf/dm-master1.toml

@@ -4,3 +4,6 @@ initial-cluster = "master1=http://127.0.0.1:8291"
 master-addr = ":8261"
 name = "master1"
 peer-urls = "127.0.0.1:8291"
+
+[experimental]
+openapi = true

dm/tests/openapi/conf/dm-master2.toml

@@ -4,3 +4,6 @@ join = "127.0.0.1:8261"
 master-addr = ":8361"
 name = "master2"
 peer-urls = "127.0.0.1:8292"
+
+[experimental]
+openapi = true