diff --git a/dm/dm/master/config.go b/dm/dm/master/config.go index 9fae89c8bde..b35ec573645 100644 --- a/dm/dm/master/config.go +++ b/dm/dm/master/config.go @@ -90,6 +90,10 @@ func NewConfig() *Config { return cfg } +type ExperimentalFeatures struct { + OpenAPI bool `toml:"openapi"` +} + // Config is the configuration for dm-master. type Config struct { flagSet *flag.FlagSet @@ -134,6 +138,8 @@ type Config struct { printVersion bool printSampleConfig bool + + ExperimentalFeatures ExperimentalFeatures `toml:"experimental"` } func (c *Config) String() string { diff --git a/dm/dm/master/config_test.go b/dm/dm/master/config_test.go index ce3ca3d5d7e..051fb17a947 100644 --- a/dm/dm/master/config_test.go +++ b/dm/dm/master/config_test.go @@ -114,6 +114,7 @@ func (t *testConfigSuite) TestConfig(c *check.C) { c.Assert(cfg.InitialClusterState, check.Equals, embed.ClusterStateFlagNew) c.Assert(cfg.Join, check.Equals, "") c.Assert(cfg.String(), check.Matches, fmt.Sprintf("{.*master-addr\":\"%s\".*}", masterAddr)) + c.Assert(cfg.ExperimentalFeatures.OpenAPI, check.Equals, false) } } } diff --git a/dm/dm/master/dm-master.toml b/dm/dm/master/dm-master.toml index 8af5014f843..a078e070cce 100644 --- a/dm/dm/master/dm-master.toml +++ b/dm/dm/master/dm-master.toml @@ -1,8 +1,8 @@ # Master Configuration. # log configuration -log-level = "info" log-file = "dm-master.log" +log-level = "info" # dm-master listen address master-addr = ":8261" @@ -39,5 +39,9 @@ rpc-timeout = "30s" # refilled at rate `rpc-rate-limit` tokens per second. Note `rpc-rate-limit` # is float64 type, so remember to add a decimal point and one trailing 0 if its # literal value happens to be an integer. -rpc-rate-limit = 10.0 rpc-rate-burst = 40 +rpc-rate-limit = 10.0 + +# some experimental features +[experimental] +openapi = false diff --git a/dm/dm/master/openapi_test.go b/dm/dm/master/openapi_test.go index 816ad8f3af8..f764d271684 100644 --- a/dm/dm/master/openapi_test.go +++ b/dm/dm/master/openapi_test.go @@ -88,6 +88,7 @@ func (t *openAPISuite) TestRedirectRequestToLeader(c *check.C) { cfg1.PeerUrls = tempurl.Alloc() cfg1.AdvertisePeerUrls = cfg1.PeerUrls cfg1.InitialCluster = fmt.Sprintf("%s=%s", cfg1.Name, cfg1.AdvertisePeerUrls) + cfg1.ExperimentalFeatures.OpenAPI = true s1 := NewServer(cfg1) c.Assert(s1.Start(ctx), check.IsNil) @@ -107,6 +108,7 @@ func (t *openAPISuite) TestRedirectRequestToLeader(c *check.C) { cfg2.PeerUrls = tempurl.Alloc() cfg2.AdvertisePeerUrls = cfg2.PeerUrls cfg2.Join = cfg1.MasterAddr // join to an existing cluster + cfg2.ExperimentalFeatures.OpenAPI = true s2 := NewServer(cfg2) c.Assert(s2.Start(ctx), check.IsNil) @@ -129,6 +131,29 @@ func (t *openAPISuite) TestRedirectRequestToLeader(c *check.C) { cancel() } +func (t *openAPISuite) TestOpenAPIWillNotStartInDefaultConfig(c *check.C) { + // create a new cluster + cfg1 := NewConfig() + c.Assert(cfg1.Parse([]string{"-config=./dm-master.toml"}), check.IsNil) + cfg1.Name = "dm-master-1" + cfg1.DataDir = c.MkDir() + cfg1.MasterAddr = tempurl.Alloc()[len("http://"):] + cfg1.PeerUrls = tempurl.Alloc() + cfg1.AdvertisePeerUrls = cfg1.PeerUrls + cfg1.InitialCluster = fmt.Sprintf("%s=%s", cfg1.Name, cfg1.AdvertisePeerUrls) + + s1 := NewServer(cfg1) + ctx, cancel := context.WithCancel(context.Background()) + c.Assert(s1.Start(ctx), check.IsNil) + // wait the first one become the leader + c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { + return s1.election.IsLeader() && s1.scheduler.Started() + }), check.IsTrue) + c.Assert(s1.echo, check.IsNil) + defer s1.Close() + cancel() +} + func (t *openAPISuite) TestSourceAPI(c *check.C) { ctx, cancel := context.WithCancel(context.Background()) s := setupServer(ctx, c) @@ -600,6 +625,7 @@ func setupServer(ctx context.Context, c *check.C) *Server { cfg1.AdvertisePeerUrls = cfg1.PeerUrls cfg1.AdvertiseAddr = cfg1.MasterAddr cfg1.InitialCluster = fmt.Sprintf("%s=%s", cfg1.Name, cfg1.AdvertisePeerUrls) + cfg1.ExperimentalFeatures.OpenAPI = true s1 := NewServer(cfg1) c.Assert(s1.Start(ctx), check.IsNil) diff --git a/dm/dm/master/server.go b/dm/dm/master/server.go index 074d63535ff..c4306158f89 100644 --- a/dm/dm/master/server.go +++ b/dm/dm/master/server.go @@ -184,15 +184,17 @@ func (s *Server) Start(ctx context.Context) (err error) { // And curl or safari browser does trigger this problem. // But I haven't figured it out. // (maybe more requests are sent from chrome or its extensions). - initOpenAPIErr := s.InitOpenAPIHandles() - if initOpenAPIErr != nil { - return terror.ErrOpenAPICommonError.Delegate(initOpenAPIErr) - } + userHandles := map[string]http.Handler{ - "/apis/": apiHandler, - "/status": getStatusHandle(), - "/debug/": getDebugHandler(), - "/api/v1/": s.echo, + "/apis/": apiHandler, + "/status": getStatusHandle(), + "/debug/": getDebugHandler(), + } + if s.cfg.ExperimentalFeatures.OpenAPI { + if initOpenAPIErr := s.InitOpenAPIHandles(); initOpenAPIErr != nil { + return terror.ErrOpenAPICommonError.Delegate(initOpenAPIErr) + } + userHandles["/api/v1/"] = s.echo } // gRPC API server diff --git a/dm/tests/dmctl_basic/conf/get_master1.toml b/dm/tests/dmctl_basic/conf/get_master1.toml index 693f3006102..2a6f1029254 100644 --- a/dm/tests/dmctl_basic/conf/get_master1.toml +++ b/dm/tests/dmctl_basic/conf/get_master1.toml @@ -24,3 +24,6 @@ v1-sources-path = "" ssl-ca = "" ssl-cert = "" ssl-key = "" + +[experimental] + openapi = false diff --git a/dm/tests/openapi/client/openapi_task_check b/dm/tests/openapi/client/openapi_task_check index 2cb936e3b5f..99136ed1423 100755 --- a/dm/tests/openapi/client/openapi_task_check +++ b/dm/tests/openapi/client/openapi_task_check @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- import sys import requests @@ -158,6 +159,20 @@ def get_task_status_failed(task_name): assert resp.status_code == 400 +def get_illegal_char_task_status_failed(): + # task name contains illegal char but api server can handle it. + # return 400 is because of the task is not started. + url = API_ENDPOINT + "/" + ILLEGAL_CHAR_TASK_NAME + "/status" + resp = requests.get(url=url) + print("get_illegal_char_task_status_failed resp=", resp.json()) + assert resp.status_code == 400 + if sys.version_info.major == 2: + # need decode in python2 + assert ILLEGAL_CHAR_TASK_NAME.decode("utf-8") in resp.json()["error_msg"] + else: + assert ILLEGAL_CHAR_TASK_NAME in resp.json()["error_msg"] + + def get_task_status_success(task_name, total): url = API_ENDPOINT + "/" + task_name + "/status" resp = requests.get(url=url) diff --git a/dm/tests/openapi/conf/dm-master1.toml b/dm/tests/openapi/conf/dm-master1.toml index e75b466b960..3ba09ab9dbc 100644 --- a/dm/tests/openapi/conf/dm-master1.toml +++ b/dm/tests/openapi/conf/dm-master1.toml @@ -4,3 +4,6 @@ initial-cluster = "master1=http://127.0.0.1:8291" master-addr = ":8261" name = "master1" peer-urls = "127.0.0.1:8291" + +[experimental] +openapi = true diff --git a/dm/tests/openapi/conf/dm-master2.toml b/dm/tests/openapi/conf/dm-master2.toml index bd2070a6d69..9b6333daa09 100644 --- a/dm/tests/openapi/conf/dm-master2.toml +++ b/dm/tests/openapi/conf/dm-master2.toml @@ -4,3 +4,6 @@ join = "127.0.0.1:8261" master-addr = ":8361" name = "master2" peer-urls = "127.0.0.1:8292" + +[experimental] +openapi = true