From fecd182ee78f2f124b240f9ad31b0d8f2353d1a6 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 21 Aug 2019 23:48:35 -0600 Subject: [PATCH 01/80] Add service.proxy.config types --- agent/config/config.go | 1 + agent/xds/config.go | 53 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/agent/config/config.go b/agent/config/config.go index 50deea2cb878..8fbc20db6429 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -87,6 +87,7 @@ func Parse(data string, format string) (c Config, err error) { "watches", "service.connect.proxy.config.upstreams", // Deprecated "services.connect.proxy.config.upstreams", // Deprecated + "services.connect.proxy.config.expose.paths", "service.connect.proxy.upstreams", "services.connect.proxy.upstreams", "service.proxy.upstreams", diff --git a/agent/xds/config.go b/agent/xds/config.go index b9894f5cb61c..d415fee0bbb0 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -139,7 +139,7 @@ func ParseUpstreamConfigNoDefaults(m map[string]interface{}) (UpstreamConfig, er return cfg, err } -// ParseUpstreamConfig returns the UpstreamConfig parsed from the an opaque map. +// ParseUpstreamConfig returns the UpstreamConfig parsed from an opaque map. // If an error occurs during parsing it is returned along with the default // config this allows caller to choose whether and how to report the error. func ParseUpstreamConfig(m map[string]interface{}) (UpstreamConfig, error) { @@ -155,3 +155,54 @@ func ParseUpstreamConfig(m map[string]interface{}) (UpstreamConfig, error) { } return cfg, err } + +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks bool `mapstructure:"checks"` + + // Port defines the port of the proxy's listener for exposed paths. + Port int `mapstructure:"port"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `mapstructure:"paths"` +} + +type Path struct { + // Path is the path to expose through the proxy, ie. "/metrics." + Path string `mapstructure:"path"` + + // Port is the port that the service is listening on for the given path. + Port int `mapstructure:"port"` + + // Protocol describes the upstream's service protocol. + // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". + Protocol string `mapstructure:"protocol"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify bool `mapstructure:"tls_skip_verify"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile string `mapstructure:"ca_file"` +} + +// ParseExposeConfig returns the ExposeConfig parsed from an opaque map. +// If an error occurs during parsing it is returned along with the default +// config this allows caller to choose whether and how to report the error. +func ParseExposeConfig(m map[string]interface{}) (ExposeConfig, error) { + var cfg ExposeConfig + err := mapstructure.WeakDecode(m, &cfg) + if cfg.Port == 0 { + cfg.Port = 21500 + } + for _, path := range cfg.Paths { + if path.Protocol == "" { + path.Protocol = "http1.1" + } else { + path.Protocol = strings.ToLower(path.Protocol) + } + } + return cfg, err +} From 9db1c109fc1527486beb173afd6b8651b4663c94 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 27 Aug 2019 23:43:37 -0600 Subject: [PATCH 02/80] Add cache-type for service HTTP checks --- agent/agent.go | 81 ++++++++++++-- agent/agent_endpoint.go | 2 +- agent/cache-types/service_checks.go | 158 ++++++++++++++++++++++++++++ agent/checks/check.go | 48 +++++++-- agent/local/state.go | 4 +- agent/service_checks_test.go | 152 ++++++++++++++++++++++++++ agent/structs/structs.go | 8 ++ 7 files changed, 429 insertions(+), 24 deletions(-) create mode 100644 agent/cache-types/service_checks.go create mode 100644 agent/service_checks_test.go diff --git a/agent/agent.go b/agent/agent.go index 74c63c89334b..255615c7b460 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -20,7 +20,7 @@ import ( "google.golang.org/grpc" - metrics "github.com/armon/go-metrics" + "github.com/armon/go-metrics" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/ae" "github.com/hashicorp/consul/agent/cache" @@ -42,8 +42,8 @@ import ( "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/consul/types" - multierror "github.com/hashicorp/go-multierror" - uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-uuid" "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/hashicorp/serf/serf" @@ -1806,7 +1806,7 @@ func (a *Agent) ResumeSync() { // syncPausedCh returns either a channel or nil. If nil sync is not paused. If // non-nil, the channel will be closed when sync resumes. -func (a *Agent) syncPausedCh() <-chan struct{} { +func (a *Agent) SyncPausedCh() <-chan struct{} { a.syncMu.Lock() defer a.syncMu.Unlock() return a.syncCh @@ -2076,7 +2076,7 @@ func (a *Agent) addServiceInternal(service *structs.NodeService, chkTypes []*str } // cleanup, store the ids of services and checks that weren't previously - // registered so we clean them up if somthing fails halfway through the + // registered so we clean them up if something fails halfway through the // process. var cleanupServices []string var cleanupChecks []types.CheckID @@ -2367,6 +2367,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, ttl := &checks.CheckTTL{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, TTL: chkType.TTL, Logger: a.logger, OutputMaxSize: maxOutputSize, @@ -2397,6 +2398,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, http := &checks.CheckHTTP{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, HTTP: chkType.HTTP, Header: chkType.Header, Method: chkType.Method, @@ -2421,12 +2423,13 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } tcp := &checks.CheckTCP{ - Notify: a.State, - CheckID: check.CheckID, - TCP: chkType.TCP, - Interval: chkType.Interval, - Timeout: chkType.Timeout, - Logger: a.logger, + Notify: a.State, + CheckID: check.CheckID, + ServiceID: check.ServiceID, + TCP: chkType.TCP, + Interval: chkType.Interval, + Timeout: chkType.Timeout, + Logger: a.logger, } tcp.Start() a.checkTCPs[check.CheckID] = tcp @@ -2450,6 +2453,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, grpc := &checks.CheckGRPC{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, GRPC: chkType.GRPC, Interval: chkType.Interval, Timeout: chkType.Timeout, @@ -2483,6 +2487,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, dockerCheck := &checks.CheckDocker{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, DockerContainerID: chkType.DockerContainerID, Shell: chkType.Shell, ScriptArgs: chkType.ScriptArgs, @@ -2509,6 +2514,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, monitor := &checks.CheckMonitor{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, ScriptArgs: chkType.ScriptArgs, Interval: chkType.Interval, Timeout: chkType.Timeout, @@ -2551,6 +2557,16 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, return fmt.Errorf("Check type is not valid") } + // Notify channel that watches for service state changes + // This is a non-blocking send to avoid synchronizing on a large number of check updates + s := a.State.ServiceState(check.ServiceID) + if s != nil && !s.Deleted { + select { + case s.WatchCh <- struct{}{}: + default: + } + } + if chkType.DeregisterCriticalServiceAfter > 0 { timeout := chkType.DeregisterCriticalServiceAfter if timeout < a.config.CheckDeregisterIntervalMin { @@ -2583,6 +2599,22 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return fmt.Errorf("CheckID missing") } + // Notify channel that watches for service state changes + // This is a non-blocking send to avoid synchronizing on a large number of check updates + var svcID string + for _, c := range a.State.Checks() { + if c.CheckID == checkID { + svcID = c.ServiceID + } + } + s := a.State.ServiceState(svcID) + if s != nil && !s.Deleted { + select { + case s.WatchCh <- struct{}{}: + default: + } + } + a.cancelCheckMonitors(checkID) a.State.RemoveCheck(checkID) @@ -2598,6 +2630,21 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return nil } +func (a *Agent) ServiceHTTPChecks(serviceID string) []structs.CheckType { + var chkTypes = make([]structs.CheckType, 0) + for _, c := range a.checkHTTPs { + if c.ServiceID == serviceID { + chkTypes = append(chkTypes, c.CheckType()) + } + } + for _, c := range a.checkGRPCs { + if c.ServiceID == serviceID { + chkTypes = append(chkTypes, c.CheckType()) + } + } + return chkTypes +} + // resolveProxyCheckAddress returns the best address to use for a TCP check of // the proxy's public listener. It expects the input to already have default // values populated by applyProxyConfigDefaults. It may return an empty string @@ -3465,4 +3512,16 @@ func (a *Agent) registerCache() { RefreshTimer: 0 * time.Second, RefreshTimeout: 10 * time.Minute, }) + + a.cache.RegisterType(cachetype.ServiceHTTPChecksName, &cachetype.ServiceHTTPChecks{ + Agent: a, + }, &cache.RegisterOptions{ + Refresh: true, + RefreshTimer: 0 * time.Second, + RefreshTimeout: 10 * time.Minute, + }) +} + +func (a *Agent) LocalState() *local.State { + return a.State } diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 6653704025ca..c7a887e465b7 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -1344,7 +1344,7 @@ func (s *HTTPServer) agentLocalBlockingQuery(resp http.ResponseWriter, hash stri // case it's likely that local state just got unloaded and may or may not be // reloaded yet. Wait a short amount of time for Sync to resume to ride out // typical config reloads. - if syncPauseCh := s.agent.syncPausedCh(); syncPauseCh != nil { + if syncPauseCh := s.agent.SyncPausedCh(); syncPauseCh != nil { select { case <-syncPauseCh: case <-timeout.C: diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go new file mode 100644 index 000000000000..bd20c3959200 --- /dev/null +++ b/agent/cache-types/service_checks.go @@ -0,0 +1,158 @@ +package cachetype + +import ( + "fmt" + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/agent/local" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/go-memdb" + "github.com/mitchellh/hashstructure" + "strings" + "time" +) + +// Recommended name for registration. +const ServiceHTTPChecksName = "service-http-checks" + +type Agent interface { + ServiceHTTPChecks(id string) []structs.CheckType + LocalState() *local.State + SyncPausedCh() <-chan struct{} +} + +// ServiceHTTPChecks supports fetching discovering checks in the local state +type ServiceHTTPChecks struct { + Agent Agent +} + +func (c *ServiceHTTPChecks) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { + var result cache.FetchResult + + // The request should be a CatalogDatacentersRequest. + reqReal, ok := req.(*ServiceHTTPChecksRequest) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: got wrong request type: %T, want: ServiceHTTPChecksRequest", req) + } + + var lastChecks *[]structs.CheckType + var lastHash string + var err error + + // Hash last known result as a baseline + if opts.LastResult != nil { + lastChecks, ok = opts.LastResult.Value.(*[]structs.CheckType) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: got wrong request type: %T, want: ServiceHTTPChecksRequest", req) + } + lastHash, err = hashChecks(*lastChecks) + if err != nil { + return result, fmt.Errorf("Internal cache failure: %v", err) + } + } + + var wait time.Duration + + // Adjust wait based on documented limits: https://www.consul.io/api/features/blocking.html + switch wait = reqReal.MaxQueryTime; { + case wait == 0*time.Second: + wait = 5 * time.Minute + case wait > 10*time.Minute: + wait = 10 * time.Minute + } + timeout := time.NewTimer(wait + lib.RandomStagger(wait/16)) + + var resp []structs.CheckType + var hash string + +WATCH_LOOP: + for { + // Must reset this every loop in case the Watch set is already closed but + // hash remains same. In that case we'll need to re-block on ws.Watch() + ws := memdb.NewWatchSet() + + svcState := c.Agent.LocalState().ServiceState(reqReal.ServiceID) + if svcState == nil { + return result, fmt.Errorf("Internal cache failure: service '%s' not in agent state", reqReal.ServiceID) + } + + // WatchCh will receive updates on service (de)registrations and check (de)registrations + ws.Add(svcState.WatchCh) + + resp = c.Agent.ServiceHTTPChecks(reqReal.ServiceID) + + hash, err := hashChecks(resp) + if err != nil { + return result, fmt.Errorf("Internal cache failure: %v", err) + } + + // Return immediately if the hash is different or the Watch returns true (indicating timeout fired). + if lastHash != hash || ws.Watch(timeout.C) { + break + } + + // Watch returned false indicating a change was detected, loop and repeat + // the call to ServiceHTTPChecks to load the new value. + // If agent sync is paused it means local state is being bulk-edited e.g. config reload. + if syncPauseCh := c.Agent.SyncPausedCh(); syncPauseCh != nil { + // Wait for pause to end or for the timeout to elapse. + select { + case <-syncPauseCh: + case <-timeout.C: + break WATCH_LOOP + } + } + } + + result.Value = &resp + + // Below is a purely synthetic index to keep the caching happy. + if opts.LastResult == nil { + result.Index = 1 + return result, nil + } + + result.Index = opts.LastResult.Index + if lastHash == "" || hash != lastHash { + result.Index += 1 + } + return result, nil +} + +func (c *ServiceHTTPChecks) SupportsBlocking() bool { + return true +} + +// ServiceHTTPChecksRequest is the cache.Request implementation for the +// ServiceHTTPChecks cache type. This is implemented here and not in structs +// since this is only used for cache-related requests and not forwarded +// directly to any Consul servers. +type ServiceHTTPChecksRequest struct { + ServiceID string + MinQueryIndex uint64 + MaxQueryTime time.Duration +} + +func (s *ServiceHTTPChecksRequest) CacheInfo() cache.RequestInfo { + return cache.RequestInfo{ + Token: "", + Key: ServiceHTTPChecksName + ":" + s.ServiceID, + Datacenter: "", + MinIndex: s.MinQueryIndex, + Timeout: s.MaxQueryTime, + } +} + +func hashChecks(checks []structs.CheckType) (string, error) { + var b strings.Builder + for _, check := range checks { + raw, err := hashstructure.Hash(check, nil) + if err != nil { + return "", fmt.Errorf("failed to hash check '%s': %v", check.CheckID, err) + } + fmt.Fprintf(&b, "%x", raw) + } + return b.String(), nil +} diff --git a/agent/checks/check.go b/agent/checks/check.go index a0816eb1246e..8a8a69352dff 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -3,6 +3,7 @@ package checks import ( "crypto/tls" "fmt" + "github.com/hashicorp/consul/agent/structs" "io" "io/ioutil" "log" @@ -58,6 +59,7 @@ type CheckNotifier interface { type CheckMonitor struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string Script string ScriptArgs []string Interval time.Duration @@ -210,10 +212,11 @@ func (c *CheckMonitor) check() { // but upon the TTL expiring, the check status is // automatically set to critical. type CheckTTL struct { - Notify CheckNotifier - CheckID types.CheckID - TTL time.Duration - Logger *log.Logger + Notify CheckNotifier + CheckID types.CheckID + ServiceID string + TTL time.Duration + Logger *log.Logger timer *time.Timer @@ -308,6 +311,7 @@ func (c *CheckTTL) SetStatus(status, output string) string { type CheckHTTP struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string HTTP string Header map[string][]string Method string @@ -323,6 +327,18 @@ type CheckHTTP struct { stopLock sync.Mutex } +func (c *CheckHTTP) CheckType() structs.CheckType { + return structs.CheckType{ + CheckID: c.CheckID, + HTTP: c.HTTP, + Method: c.Method, + Header: c.Header, + Interval: c.Interval, + Timeout: c.Timeout, + OutputMaxSize: c.OutputMaxSize, + } +} + // Start is used to start an HTTP check. // The check runs until stop is called func (c *CheckHTTP) Start() { @@ -456,12 +472,13 @@ func (c *CheckHTTP) check() { // The check is passing if the connection succeeds // The check is critical if the connection returns an error type CheckTCP struct { - Notify CheckNotifier - CheckID types.CheckID - TCP string - Interval time.Duration - Timeout time.Duration - Logger *log.Logger + Notify CheckNotifier + CheckID types.CheckID + ServiceID string + TCP string + Interval time.Duration + Timeout time.Duration + Logger *log.Logger dialer *net.Dialer stop bool @@ -537,6 +554,7 @@ func (c *CheckTCP) check() { type CheckDocker struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string Script string ScriptArgs []string DockerContainerID string @@ -656,6 +674,7 @@ func (c *CheckDocker) doCheck() (string, *circbuf.Buffer, error) { type CheckGRPC struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string GRPC string Interval time.Duration Timeout time.Duration @@ -668,6 +687,15 @@ type CheckGRPC struct { stopLock sync.Mutex } +func (c *CheckGRPC) CheckType() structs.CheckType { + return structs.CheckType{ + CheckID: c.CheckID, + GRPC: c.GRPC, + Interval: c.Interval, + Timeout: c.Timeout, + } +} + func (c *CheckGRPC) Start() { c.stopLock.Lock() defer c.stopLock.Unlock() diff --git a/agent/local/state.go b/agent/local/state.go index 7b489782793b..dda3843f009a 100644 --- a/agent/local/state.go +++ b/agent/local/state.go @@ -50,7 +50,7 @@ type ServiceState struct { // but has not been removed on the server yet. Deleted bool - // WatchCh is closed when the service state changes suitable for use in a + // WatchCh is closed when the service state changes. Suitable for use in a // memdb.WatchSet when watching agent local changes with hash-based blocking. WatchCh chan struct{} } @@ -366,7 +366,7 @@ func (l *State) SetServiceState(s *ServiceState) { } func (l *State) setServiceStateLocked(s *ServiceState) { - s.WatchCh = make(chan struct{}) + s.WatchCh = make(chan struct{}, 1) old, hasOld := l.services[s.Service.ID] l.services[s.Service.ID] = s diff --git a/agent/service_checks_test.go b/agent/service_checks_test.go new file mode 100644 index 000000000000..083fc6570c3c --- /dev/null +++ b/agent/service_checks_test.go @@ -0,0 +1,152 @@ +package agent + +import ( + "context" + "github.com/hashicorp/consul/agent/cache" + cachetype "github.com/hashicorp/consul/agent/cache-types" + "github.com/hashicorp/consul/agent/checks" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/testrpc" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + + service := structs.NodeService{ + ID: "web", + Service: "web", + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ch := make(chan cache.UpdateEvent) + + // Watch for service check updates + err := a.cache.Notify(ctx, cachetype.ServiceHTTPChecksName, &cachetype.ServiceHTTPChecksRequest{ + ServiceID: service.ID, + }, "service-checks:"+service.ID, ch) + if err != nil { + t.Fatalf("failed to set cache notification: %v", err) + } + + chkTypes := []*structs.CheckType{ + { + CheckID: "http-check", + HTTP: "localhost:8080/health", + Interval: 5 * time.Second, + OutputMaxSize: checks.DefaultBufSize, + }, + { + CheckID: "grpc-check", + GRPC: "localhost:9090/v1.Health", + Interval: 5 * time.Second, + }, + { + CheckID: "ttl-check", + TTL: 10 * time.Second, + }, + } + + // Adding first TTL type should lead to a timeout, since only HTTP-based checks are watched + if err := a.AddService(&service, chkTypes[2:], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add service: %v", err) + } + + var val cache.UpdateEvent + select { + case val = <-ch: + t.Fatal("unexpected cache update, wanted HTTP checks, got TTL") + case <-time.After(100 * time.Millisecond): + } + + // Adding service with HTTP check should lead notification for check + if err := a.AddService(&service, chkTypes[0:1], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add service: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok := val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + want := chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Adding GRPC check should lead to a notification from the cache with both checks + hc := structs.HealthCheck{ + CheckID: chkTypes[1].CheckID, + ServiceID: service.ID, + } + if err := a.AddCheck(&hc, chkTypes[1], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add service: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok = val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + want = chkTypes[0:2] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Removing the GRPC check should leave only the HTTP check + if err := a.RemoveCheck(chkTypes[1].CheckID, false); err != nil { + t.Fatalf("failed to remove check: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok = val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + want = chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Removing the HTTP check should leave an empty list + if err := a.RemoveCheck(chkTypes[0].CheckID, false); err != nil { + t.Fatalf("failed to remove check: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok = val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + if len(*got) != 0 { + t.Fatalf("expected empty result, got: %+v", got) + } +} diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 166c41bbb886..1676b85e7085 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -1122,6 +1122,14 @@ type HealthCheckDefinition struct { OutputMaxSize uint `json:",omitempty"` Timeout time.Duration `json:",omitempty"` DeregisterCriticalServiceAfter time.Duration `json:",omitempty"` + ScriptArgs []string `json:",omitempty"` + DockerContainerID string `json:",omitempty"` + Shell string `json:",omitempty"` + GRPC string `json:",omitempty"` + GRPCUseTLS bool `json:",omitempty"` + AliasNode string `json:",omitempty"` + AliasService string `json:",omitempty"` + TTL time.Duration `json:",omitempty"` } func (d *HealthCheckDefinition) MarshalJSON() ([]byte, error) { From 72ead604ce9807b703e6e79c4e39a8b9709eac7d Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 28 Aug 2019 23:01:44 -0600 Subject: [PATCH 03/80] Update tests for service-checks cachetype and fix hash bug --- agent/cache-types/service_checks.go | 4 +- agent/cache-types/service_checks_test.go | 178 +++++++++++++++++++++++ agent/service_checks_test.go | 57 +------- 3 files changed, 188 insertions(+), 51 deletions(-) create mode 100644 agent/cache-types/service_checks_test.go diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go index bd20c3959200..2ddae4ad68a7 100644 --- a/agent/cache-types/service_checks.go +++ b/agent/cache-types/service_checks.go @@ -55,7 +55,7 @@ func (c *ServiceHTTPChecks) Fetch(opts cache.FetchOptions, req cache.Request) (c var wait time.Duration - // Adjust wait based on documented limits: https://www.consul.io/api/features/blocking.html + // Adjust wait based on documented limits and add some jitter: https://www.consul.io/api/features/blocking.html switch wait = reqReal.MaxQueryTime; { case wait == 0*time.Second: wait = 5 * time.Minute @@ -83,7 +83,7 @@ WATCH_LOOP: resp = c.Agent.ServiceHTTPChecks(reqReal.ServiceID) - hash, err := hashChecks(resp) + hash, err = hashChecks(resp) if err != nil { return result, fmt.Errorf("Internal cache failure: %v", err) } diff --git a/agent/cache-types/service_checks_test.go b/agent/cache-types/service_checks_test.go new file mode 100644 index 000000000000..133a80cb9a45 --- /dev/null +++ b/agent/cache-types/service_checks_test.go @@ -0,0 +1,178 @@ +package cachetype_test + +import ( + "github.com/hashicorp/consul/agent/cache" + cachetype "github.com/hashicorp/consul/agent/cache-types" + "github.com/hashicorp/consul/agent/checks" + "github.com/hashicorp/consul/agent/local" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/token" + "github.com/hashicorp/consul/types" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestServiceHTTPChecks_Fetch(t *testing.T) { + chkTypes := []*structs.CheckType{ + { + CheckID: "http-check", + HTTP: "localhost:8080/health", + Interval: 5 * time.Second, + OutputMaxSize: checks.DefaultBufSize, + }, + { + CheckID: "grpc-check", + GRPC: "localhost:9090/v1.Health", + Interval: 5 * time.Second, + }, + } + + svcState := local.ServiceState{ + Service: &structs.NodeService{ + ID: "web", + }, + } + + // Create mockAgent and cache type + a := newMockAgent() + a.LocalState().SetServiceState(&svcState) + typ := cachetype.ServiceHTTPChecks{Agent: a} + + // Adding HTTP check should yield check in Fetch + if err := a.AddCheck(*chkTypes[0]); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result, err := typ.Fetch( + cache.FetchOptions{}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result.Index != 1 { + t.Fatalf("expected index of 1 after first request, got %d", result.Index) + } + + got, ok := result.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want := chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Adding GRPC check should yield both checks in Fetch + if err := a.AddCheck(*chkTypes[1]); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result2, err := typ.Fetch( + cache.FetchOptions{LastResult: &result}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result2.Index != 2 { + t.Fatalf("expected index of 2 after second request, got %d", result2.Index) + } + + got, ok = result2.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want = chkTypes[0:2] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Removing GRPC check should yield HTTP check in Fetch + if err := a.RemoveCheck(chkTypes[1].CheckID); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result3, err := typ.Fetch( + cache.FetchOptions{LastResult: &result2}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result3.Index != 3 { + t.Fatalf("expected index of 3 after third request, got %d", result3.Index) + } + + got, ok = result3.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want = chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Fetching again should yield no change in result nor index + result4, err := typ.Fetch( + cache.FetchOptions{LastResult: &result3}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result4.Index != 3 { + t.Fatalf("expected index of 3 after fetch timeout, got %d", result4.Index) + } + + got, ok = result4.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want = chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } +} + +type mockAgent struct { + state *local.State + pauseCh <-chan struct{} + checks []structs.CheckType +} + +func newMockAgent() *mockAgent { + m := mockAgent{ + state: local.NewState(local.Config{NodeID: "host"}, nil, new(token.Store)), + pauseCh: make(chan struct{}), + checks: make([]structs.CheckType, 0), + } + m.state.TriggerSyncChanges = func() {} + return &m +} + +func (m *mockAgent) ServiceHTTPChecks(id string) []structs.CheckType { + return m.checks +} + +func (m *mockAgent) LocalState() *local.State { + return m.state +} + +func (m *mockAgent) SyncPausedCh() <-chan struct{} { + return m.pauseCh +} + +func (m *mockAgent) AddCheck(check structs.CheckType) error { + m.checks = append(m.checks, check) + return nil +} + +func (m *mockAgent) RemoveCheck(id types.CheckID) error { + new := make([]structs.CheckType, 0) + for _, c := range m.checks { + if c.CheckID != id { + new = append(new, c) + } + } + m.checks = new + return nil +} diff --git a/agent/service_checks_test.go b/agent/service_checks_test.go index 083fc6570c3c..b4f45ab3b5ec 100644 --- a/agent/service_checks_test.go +++ b/agent/service_checks_test.go @@ -12,6 +12,8 @@ import ( "time" ) +// Integration test for ServiceHTTPChecks cache-type +// Placed in agent pkg rather than cache-types to avoid circular dependency when importing agent.TestAgent func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { t.Parallel() @@ -63,12 +65,12 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { var val cache.UpdateEvent select { case val = <-ch: - t.Fatal("unexpected cache update, wanted HTTP checks, got TTL") + t.Fatal("got cache update for TTL check, expected timeout") case <-time.After(100 * time.Millisecond): } - // Adding service with HTTP check should lead notification for check - if err := a.AddService(&service, chkTypes[0:1], false, "", ConfigSourceLocal); err != nil { + // Adding service with HTTP checks should lead notification for them + if err := a.AddService(&service, chkTypes[0:2], false, "", ConfigSourceLocal); err != nil { t.Fatalf("failed to add service: %v", err) } @@ -80,33 +82,9 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { got, ok := val.Result.(*[]structs.CheckType) if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + t.Fatalf("notified of result of wrong type, got %T, want *[]structs.CheckType", got) } - want := chkTypes[0:1] - for i, c := range *got { - require.Equal(t, c, *want[i]) - } - - // Adding GRPC check should lead to a notification from the cache with both checks - hc := structs.HealthCheck{ - CheckID: chkTypes[1].CheckID, - ServiceID: service.ID, - } - if err := a.AddCheck(&hc, chkTypes[1], false, "", ConfigSourceLocal); err != nil { - t.Fatalf("failed to add service: %v", err) - } - - select { - case val = <-ch: - case <-time.After(100 * time.Millisecond): - t.Fatal("didn't get cache update event") - } - - got, ok = val.Result.(*[]structs.CheckType) - if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) - } - want = chkTypes[0:2] + want := chkTypes[0:2] for i, c := range *got { require.Equal(t, c, *want[i]) } @@ -124,29 +102,10 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { got, ok = val.Result.(*[]structs.CheckType) if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + t.Fatalf("notified of result of wrong type, got %T, want *[]structs.CheckType", got) } want = chkTypes[0:1] for i, c := range *got { require.Equal(t, c, *want[i]) } - - // Removing the HTTP check should leave an empty list - if err := a.RemoveCheck(chkTypes[0].CheckID, false); err != nil { - t.Fatalf("failed to remove check: %v", err) - } - - select { - case val = <-ch: - case <-time.After(100 * time.Millisecond): - t.Fatal("didn't get cache update event") - } - - got, ok = val.Result.(*[]structs.CheckType) - if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) - } - if len(*got) != 0 { - t.Fatalf("expected empty result, got: %+v", got) - } } From 011ac9ef319e79d6b299b2c39b9f62e63b736f41 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 28 Aug 2019 23:10:09 -0600 Subject: [PATCH 04/80] Set up service-check notification and handling --- agent/proxycfg/snapshot.go | 1 + agent/proxycfg/state.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 37b8bf574958..47ee197858ed 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -14,6 +14,7 @@ type configSnapshotConnectProxy struct { WatchedUpstreamEndpoints map[string]map[string]structs.CheckServiceNodes WatchedGateways map[string]map[string]context.CancelFunc WatchedGatewayEndpoints map[string]map[string]structs.CheckServiceNodes + WatchedServiceChecks map[string][]structs.CheckType UpstreamEndpoints map[string]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints } diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index eea31d100ff4..608c4d20915d 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -29,6 +29,7 @@ const ( serviceListWatchID = "service-list" datacentersWatchID = "datacenters" serviceResolversWatchID = "service-resolvers" + svcChecksWatchIDPrefix = cachetype.ServiceHTTPChecksName + ":" serviceIDPrefix = string(structs.UpstreamDestTypeService) + ":" preparedQueryIDPrefix = string(structs.UpstreamDestTypePreparedQuery) + ":" defaultPreparedQueryPollInterval = 30 * time.Second @@ -215,6 +216,14 @@ func (s *state) initWatchesConnectProxy() error { return err } + // Watch for service check updates + err = s.cache.Notify(s.ctx, cachetype.ServiceHTTPChecksName, &cachetype.ServiceHTTPChecksRequest{ + ServiceID: s.proxyCfg.DestinationServiceID, + }, svcChecksWatchIDPrefix+s.proxyCfg.DestinationServiceID, s.ch) + if err != nil { + return err + } + // TODO(namespaces): pull this from something like s.source.Namespace? currentNamespace := "default" @@ -353,6 +362,7 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot { snap.ConnectProxy.WatchedUpstreamEndpoints = make(map[string]map[string]structs.CheckServiceNodes) snap.ConnectProxy.WatchedGateways = make(map[string]map[string]context.CancelFunc) snap.ConnectProxy.WatchedGatewayEndpoints = make(map[string]map[string]structs.CheckServiceNodes) + snap.ConnectProxy.WatchedServiceChecks = make(map[string][]structs.CheckType) snap.ConnectProxy.UpstreamEndpoints = make(map[string]structs.CheckServiceNodes) // TODO(rb): deprecated case structs.ServiceKindMeshGateway: @@ -541,6 +551,14 @@ func (s *state) handleUpdateConnectProxy(u cache.UpdateEvent, snap *ConfigSnapsh pq := strings.TrimPrefix(u.CorrelationID, "upstream:") snap.ConnectProxy.UpstreamEndpoints[pq] = resp.Nodes + case strings.HasPrefix(u.CorrelationID, svcChecksWatchIDPrefix): + resp, ok := u.Result.(*[]structs.CheckType) + if !ok { + return fmt.Errorf("invalid type for service checks response: %T, want: *[]structs.CheckType", u.Result) + } + svcID := strings.TrimPrefix(u.CorrelationID, svcChecksWatchIDPrefix) + snap.ConnectProxy.WatchedServiceChecks[svcID] = *resp + default: return errors.New("unknown correlation ID") } From dc1a343d1ef512749590806c8c4f3be17d95c961 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 29 Aug 2019 16:07:11 -0600 Subject: [PATCH 05/80] Move expose config out of opaque proxy.config --- agent/config/config.go | 1 - agent/structs/connect_proxy_config.go | 77 +++++++++++++++++++++++++++ agent/xds/config.go | 51 ------------------ 3 files changed, 77 insertions(+), 52 deletions(-) diff --git a/agent/config/config.go b/agent/config/config.go index 8fbc20db6429..50deea2cb878 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -87,7 +87,6 @@ func Parse(data string, format string) (c Config, err error) { "watches", "service.connect.proxy.config.upstreams", // Deprecated "services.connect.proxy.config.upstreams", // Deprecated - "services.connect.proxy.config.expose.paths", "service.connect.proxy.upstreams", "services.connect.proxy.upstreams", "service.proxy.upstreams", diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 78c4cb93635a..0f92f332bf85 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -3,11 +3,20 @@ package structs import ( "encoding/json" "fmt" + "io/ioutil" + "strings" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" ) +const ( + defaultExposePort = 21500 + defaultExposeProtocol = "http1.1" +) + +var allowedExposeProtocols = map[string]bool{"http1.1": true, "http2": true, "grpc": true} + type MeshGatewayMode string const ( @@ -312,3 +321,71 @@ func UpstreamFromAPI(u api.Upstream) Upstream { Config: u.Config, } } + +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks bool `mapstructure:"checks"` + + // Port defines the port of the proxy's listener for exposed paths. + Port int `mapstructure:"port"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `mapstructure:"paths"` +} + +type Path struct { + // Path is the path to expose through the proxy, ie. "/metrics." + Path string `mapstructure:"path"` + + // Port is the port that the service is listening on for the given path. + Port int `mapstructure:"port"` + + // Protocol describes the upstream's service protocol. + // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". + Protocol string `mapstructure:"protocol"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify bool `mapstructure:"tls_skip_verify"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile string `mapstructure:"ca_file"` + + // CACert contains the PEM encoded CA file read from CAFile + CACert string +} + +// Finalize validates ExposeConfig and sets default values +func (e *ExposeConfig) Finalize() error { + if e.Port < 0 || e.Port > 65535 { + return fmt.Errorf("invalid port: %d", e.Port) + } + if e.Port == 0 { + e.Port = defaultExposePort + } + + var known = make(map[string]bool) + for _, path := range e.Paths { + if seen := known[path.Path]; seen { + return fmt.Errorf("duplicate paths exposed") + } + known[path.Path] = true + + b, err := ioutil.ReadFile(path.CAFile) + if err != nil { + return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + } + path.CACert = string(b) + + path.Protocol = strings.ToLower(path.Protocol) + if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { + return fmt.Errorf("protocol '%s' not recognized for path: %s", path.Protocol, path.Path) + } + if path.Protocol == "" { + path.Protocol = defaultExposeProtocol + } + } + return nil +} diff --git a/agent/xds/config.go b/agent/xds/config.go index d415fee0bbb0..b3df755cd6cd 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -155,54 +155,3 @@ func ParseUpstreamConfig(m map[string]interface{}) (UpstreamConfig, error) { } return cfg, err } - -// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. -// Users can expose individual paths and/or all HTTP/GRPC paths for checks. -type ExposeConfig struct { - // Checks defines whether paths associated with Consul checks will be exposed. - // This flag triggers exposing all HTTP and GRPC check paths registered for the service. - Checks bool `mapstructure:"checks"` - - // Port defines the port of the proxy's listener for exposed paths. - Port int `mapstructure:"port"` - - // Paths is the list of paths exposed through the proxy. - Paths []Path `mapstructure:"paths"` -} - -type Path struct { - // Path is the path to expose through the proxy, ie. "/metrics." - Path string `mapstructure:"path"` - - // Port is the port that the service is listening on for the given path. - Port int `mapstructure:"port"` - - // Protocol describes the upstream's service protocol. - // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". - Protocol string `mapstructure:"protocol"` - - // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. - TLSSkipVerify bool `mapstructure:"tls_skip_verify"` - - // CAFile is the path to the PEM encoded CA cert used to verify client certificates. - CAFile string `mapstructure:"ca_file"` -} - -// ParseExposeConfig returns the ExposeConfig parsed from an opaque map. -// If an error occurs during parsing it is returned along with the default -// config this allows caller to choose whether and how to report the error. -func ParseExposeConfig(m map[string]interface{}) (ExposeConfig, error) { - var cfg ExposeConfig - err := mapstructure.WeakDecode(m, &cfg) - if cfg.Port == 0 { - cfg.Port = 21500 - } - for _, path := range cfg.Paths { - if path.Protocol == "" { - path.Protocol = "http1.1" - } else { - path.Protocol = strings.ToLower(path.Protocol) - } - } - return cfg, err -} From f9c05b7aab2161fa1a640c39f46ad138963735ed Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 30 Aug 2019 13:31:18 -0600 Subject: [PATCH 06/80] Inject proxy address into HTTP checks --- agent/agent.go | 42 ++++++++++ agent/agent_test.go | 107 ++++++++++++++++++++++++++ agent/checks/check.go | 13 +++- agent/structs/connect_proxy_config.go | 3 + 4 files changed, 163 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 255615c7b460..ea760f7c7a3c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strconv" "strings" "sync" @@ -2408,6 +2409,16 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, OutputMaxSize: maxOutputSize, TLSClientConfig: tlsClientConfig, } + + if service.Proxy.Expose.Checks { + addr, err := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which it shouldn't + return fmt.Errorf("failed to inject proxy addr into HTTP target") + } + http.ProxyHTTP = addr + } + http.Start() a.checkHTTPs[check.CheckID] = http @@ -3525,3 +3536,34 @@ func (a *Agent) registerCache() { func (a *Agent) LocalState() *local.State { return a.State } + +// httpInjectAddr injects a port then an IP into a URL +func httpInjectAddr(url string, ip string, port int) (string, error) { + pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" + r, err := regexp.Compile(pattern) + if err != nil { + return "", err + } + portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) + out := r.ReplaceAllString(url, portRepl) + + // Ensure that ipv6 addr is enclosed in brackets (RFC 3986) + ip = fixIPv6(ip) + addrRepl := fmt.Sprintf("${1}%s${3}${4}${5}", ip) + out = r.ReplaceAllString(out, addrRepl) + + return out, nil +} + +func fixIPv6(address string) string { + if strings.Count(address, ":") < 2 { + return address + } + if !strings.HasSuffix(address, "]") { + address = address + "]" + } + if !strings.HasPrefix(address, "[") { + address = "[" + address + } + return address +} diff --git a/agent/agent_test.go b/agent/agent_test.go index 06f82d6c8a58..62f060ca8548 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3350,3 +3350,110 @@ func TestAgent_consulConfig_RaftTrailingLogs(t *testing.T) { defer a.Shutdown() require.Equal(t, uint64(812345), a.consulConfig().RaftConfig.TrailingLogs) } + +func TestAgent_httpInjectAddr(t *testing.T) { + tt := []struct { + name string + url string + ip string + port int + want string + }{ + // TODO(freddy): IPv6 checks + { + name: "localhost health", + url: "http://localhost:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "http://192.168.0.0:9090/health", + }, + { + name: "https localhost health", + url: "https://localhost:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv4 health", + url: "https://127.0.0.1:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv4 without path", + url: "https://127.0.0.1:8080", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090", + }, + { + name: "https ipv6 health", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv6 with zone", + url: "https://[::FFFF:C0A8:1%1]:5000/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv6 literal", + url: "https://[::FFFF:192.168.0.1]:5000/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv6 without path", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090", + }, + { + name: "ipv6 injected into ipv6 url", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "https://[::FFFF:C0A8:1]:9090", + }, + { + name: "ipv6 with brackets injected into ipv6 url", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000", + ip: "[::FFFF:C0A8:1]", + port: 9090, + want: "https://[::FFFF:C0A8:1]:9090", + }, + { + name: "short domain health", + url: "http://i.co:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "http://192.168.0.0:9090/health", + }, + { + name: "nested url in query", + url: "http://my.corp.com:8080/health?from=http://google.com:8080", + ip: "192.168.0.0", + port: 9090, + want: "http://192.168.0.0:9090/health?from=http://google.com:8080", + }, + } + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + got, err := httpInjectAddr(tt.url, tt.ip, tt.port) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/agent/checks/check.go b/agent/checks/check.go index 8a8a69352dff..03b8fd379183 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -325,6 +325,10 @@ type CheckHTTP struct { stop bool stopCh chan struct{} stopLock sync.Mutex + + // Set if checks are exposed through Connect proxies + // If set, this is the target of check() + ProxyHTTP string } func (c *CheckHTTP) CheckType() structs.CheckType { @@ -406,7 +410,12 @@ func (c *CheckHTTP) check() { method = "GET" } - req, err := http.NewRequest(method, c.HTTP, nil) + target := c.HTTP + if c.ProxyHTTP != "" { + target = c.HTTP + } + + req, err := http.NewRequest(method, target, nil) if err != nil { c.Logger.Printf("[WARN] agent: Check %q HTTP request failed: %s", c.CheckID, err) c.Notify.UpdateCheck(c.CheckID, api.HealthCritical, err.Error()) @@ -446,7 +455,7 @@ func (c *CheckHTTP) check() { } // Format the response body - result := fmt.Sprintf("HTTP %s %s: %s Output: %s", method, c.HTTP, resp.Status, output.String()) + result := fmt.Sprintf("HTTP %s %s: %s Output: %s", method, target, resp.Status, output.String()) if resp.StatusCode >= 200 && resp.StatusCode <= 299 { // PASSING (2xx) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 0f92f332bf85..57ad153bd27d 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -118,6 +118,9 @@ type ConnectProxyConfig struct { // MeshGateway defines the mesh gateway configuration for this upstream MeshGateway MeshGatewayConfig `json:",omitempty"` + + // Expose defines whether checks or paths are exposed through the proxy + Expose ExposeConfig `json:",omitempty"` } func (c *ConnectProxyConfig) MarshalJSON() ([]byte, error) { From 839de82c6d64b7eede39bab5b27b6a0733740ddd Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 30 Aug 2019 17:26:24 -0600 Subject: [PATCH 07/80] Inject proxy address into GRPC checks --- agent/agent.go | 28 +++++++++++- agent/agent_test.go | 100 +++++++++++++++++++++++++++++++++++++++++- agent/checks/check.go | 13 +++++- agent/checks/grpc.go | 9 ++-- 4 files changed, 142 insertions(+), 8 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index ea760f7c7a3c..a9ed5200383a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2413,7 +2413,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, if service.Proxy.Expose.Checks { addr, err := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which it shouldn't + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests return fmt.Errorf("failed to inject proxy addr into HTTP target") } http.ProxyHTTP = addr @@ -2471,6 +2471,16 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, Logger: a.logger, TLSClientConfig: tlsClientConfig, } + + if service.Proxy.Expose.Checks { + addr, err := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests + return fmt.Errorf("failed to inject proxy addr into GRPC target") + } + grpc.ProxyGRPC = addr + } + grpc.Start() a.checkGRPCs[check.CheckID] = grpc @@ -3537,6 +3547,22 @@ func (a *Agent) LocalState() *local.State { return a.State } +// grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] +func grpcInjectAddr(existing string, ip string, port int) (string, error) { + pattern := "(.*)((?::)(?:[0-9]+))(.*)$" + r, err := regexp.Compile(pattern) + if err != nil { + return "", err + } + portRepl := fmt.Sprintf("${1}:%d${3}", port) + out := r.ReplaceAllString(existing, portRepl) + + addrRepl := fmt.Sprintf("%s${2}${3}", ip) + out = r.ReplaceAllString(out, addrRepl) + + return out, nil +} + // httpInjectAddr injects a port then an IP into a URL func httpInjectAddr(url string, ip string, port int) (string, error) { pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" diff --git a/agent/agent_test.go b/agent/agent_test.go index 62f060ca8548..34f702aaf3f6 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3351,6 +3351,105 @@ func TestAgent_consulConfig_RaftTrailingLogs(t *testing.T) { require.Equal(t, uint64(812345), a.consulConfig().RaftConfig.TrailingLogs) } +func TestAgent_grpcInjectAddr(t *testing.T) { + tt := []struct { + name string + grpc string + ip string + port int + want string + }{ + { + name: "localhost web svc", + grpc: "localhost:8080/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "localhost no svc", + grpc: "localhost:8080", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090", + }, + { + name: "ipv4 web svc", + grpc: "127.0.0.1:8080/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "ipv4 no svc", + grpc: "127.0.0.1:8080", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090", + }, + { + name: "ipv6 no svc", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090", + }, + { + name: "ipv6 web svc", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "zone ipv6 web svc", + grpc: "::FFFF:C0A8:1%1:5000/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "ipv6 literal web svc", + grpc: "::FFFF:192.168.0.1:5000/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "ipv6 injected into ipv6 url", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "::FFFF:C0A8:1:9090", + }, + { + name: "ipv6 injected into ipv6 url with svc", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000/web", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "::FFFF:C0A8:1:9090/web", + }, + { + name: "ipv6 injected into ipv6 url with special", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000/service-$name:with@special:Chars", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "::FFFF:C0A8:1:9090/service-$name:with@special:Chars", + }, + } + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + got, err := grpcInjectAddr(tt.grpc, tt.ip, tt.port) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) + } + }) + } +} + func TestAgent_httpInjectAddr(t *testing.T) { tt := []struct { name string @@ -3359,7 +3458,6 @@ func TestAgent_httpInjectAddr(t *testing.T) { port int want string }{ - // TODO(freddy): IPv6 checks { name: "localhost health", url: "http://localhost:8080/health", diff --git a/agent/checks/check.go b/agent/checks/check.go index 03b8fd379183..d25566ff7ae2 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -694,6 +694,10 @@ type CheckGRPC struct { stop bool stopCh chan struct{} stopLock sync.Mutex + + // Set if checks are exposed through Connect proxies + // If set, this is the target of check() + ProxyGRPC string } func (c *CheckGRPC) CheckType() structs.CheckType { @@ -734,13 +738,18 @@ func (c *CheckGRPC) run() { } func (c *CheckGRPC) check() { - err := c.probe.Check() + target := c.GRPC + if c.ProxyGRPC != "" { + target = c.ProxyGRPC + } + + err := c.probe.Check(target) if err != nil { c.Logger.Printf("[DEBUG] agent: Check %q failed: %s", c.CheckID, err.Error()) c.Notify.UpdateCheck(c.CheckID, api.HealthCritical, err.Error()) } else { c.Logger.Printf("[DEBUG] agent: Check %q is passing", c.CheckID) - c.Notify.UpdateCheck(c.CheckID, api.HealthPassing, fmt.Sprintf("gRPC check %s: success", c.GRPC)) + c.Notify.UpdateCheck(c.CheckID, api.HealthPassing, fmt.Sprintf("gRPC check %s: success", target)) } } diff --git a/agent/checks/grpc.go b/agent/checks/grpc.go index 8577ae6e7c40..4ad7b9f34bf4 100644 --- a/agent/checks/grpc.go +++ b/agent/checks/grpc.go @@ -28,7 +28,6 @@ type GrpcHealthProbe struct { func NewGrpcHealthProbe(target string, timeout time.Duration, tlsConfig *tls.Config) *GrpcHealthProbe { serverAndService := strings.SplitN(target, "/", 2) - server := serverAndService[0] request := hv1.HealthCheckRequest{} if len(serverAndService) > 1 { request.Service = serverAndService[1] @@ -43,7 +42,6 @@ func NewGrpcHealthProbe(target string, timeout time.Duration, tlsConfig *tls.Con } return &GrpcHealthProbe{ - server: server, request: &request, timeout: timeout, dialOptions: dialOptions, @@ -52,11 +50,14 @@ func NewGrpcHealthProbe(target string, timeout time.Duration, tlsConfig *tls.Con // Check if the target of this GrpcHealthProbe is healthy // If nil is returned, target is healthy, otherwise target is not healthy -func (probe *GrpcHealthProbe) Check() error { +func (probe *GrpcHealthProbe) Check(target string) error { + serverAndService := strings.SplitN(target, "/", 2) + server := serverAndService[0] + ctx, cancel := context.WithTimeout(context.Background(), probe.timeout) defer cancel() - connection, err := grpc.DialContext(ctx, probe.server, probe.dialOptions...) + connection, err := grpc.DialContext(ctx, server, probe.dialOptions...) if err != nil { return err } From 4fa321790c23ed741d19ad0e586053134c80f380 Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 30 Aug 2019 18:19:28 -0600 Subject: [PATCH 08/80] Set and reset check targets on proxy (de)registration --- agent/agent.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/agent/agent.go b/agent/agent.go index a9ed5200383a..c5ca7a6fc4a0 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2113,6 +2113,20 @@ func (a *Agent) addServiceInternal(service *structs.NodeService, chkTypes []*str } } + // If a proxy service wishes to expose checks, check targets need to be rerouted to the proxy listener + // This needs to be called after chkTypes are added to the agent, to avoid overwriting + if service.Proxy.Expose.Checks { + err := a.rerouteExposedChecks( + service.Proxy.DestinationServiceID, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + if err != nil { + a.logger.Println("[WARN] failed to reroute L7 checks to exposed proxy listener") + } + } else { + // Reset check targets if proxy was re-registered but no longer wants to expose checks + // If the proxy is being registered for the first time then this is a no-op + a.resetExposedChecks(service.Proxy.DestinationServiceID) + } + // Persist the service to a file if persist && a.config.DataDir != "" { if err := a.persistService(service); err != nil { @@ -2224,6 +2238,11 @@ func (a *Agent) removeServiceLocked(serviceID string, persist bool) error { a.serviceManager.RemoveService(serviceID) } + // Reset the HTTP check targets if they were exposed through a proxy + // If this is not a proxy or checks were not exposed then this is a no-op + svc := a.State.Service(serviceID) + a.resetExposedChecks(svc.Proxy.DestinationServiceID) + checks := a.State.Checks() var checkIDs []types.CheckID for id, check := range checks { @@ -3547,6 +3566,54 @@ func (a *Agent) LocalState() *local.State { return a.State } +// rerouteExposedChecks will inject proxy address into check targets +// Future calls to check() will dial the proxy listener +// The agent stateLock MUST be held for this to be called +func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPort int) error { + for _, c := range a.checkHTTPs { + if c.ServiceID != serviceID { + continue + } + addr, err := httpInjectAddr(c.HTTP, proxyAddr, proxyPort) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests + return fmt.Errorf("failed to inject proxy addr into HTTP target") + } + c.ProxyHTTP = addr + } + for _, c := range a.checkGRPCs { + if c.ServiceID != serviceID { + continue + } + addr, err := grpcInjectAddr(c.GRPC, proxyAddr, proxyPort) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests + return fmt.Errorf("failed to inject proxy addr into GRPC target") + } + c.ProxyGRPC = addr + } + return nil +} + +// resetExposedChecks will set Proxy addr in HTTP checks to empty string +// Future calls to check() will use the original target c.HTTP or c.GRPC +// The agent stateLock MUST be held for this to be called +func (a *Agent) resetExposedChecks(serviceID string) { + a.stateLock.Lock() + defer a.stateLock.Unlock() + + for _, c := range a.checkHTTPs { + if c.ServiceID == serviceID { + c.ProxyHTTP = "" + } + } + for _, c := range a.checkGRPCs { + if c.ServiceID == serviceID { + c.ProxyGRPC = "" + } + } +} + // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] func grpcInjectAddr(existing string, ip string, port int) (string, error) { pattern := "(.*)((?::)(?:[0-9]+))(.*)$" From 45c0984f3647a297e43a44e4f0ef855ab479de89 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 2 Sep 2019 13:16:27 -0600 Subject: [PATCH 09/80] Rename http1.1 in Expose.Protocol to http --- agent/structs/connect_proxy_config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 57ad153bd27d..d43edc4e280f 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -12,10 +12,10 @@ import ( const ( defaultExposePort = 21500 - defaultExposeProtocol = "http1.1" + defaultExposeProtocol = "http" ) -var allowedExposeProtocols = map[string]bool{"http1.1": true, "http2": true, "grpc": true} +var allowedExposeProtocols = map[string]bool{"http": true, "http2": true, "grpc": true} type MeshGatewayMode string @@ -347,7 +347,7 @@ type Path struct { Port int `mapstructure:"port"` // Protocol describes the upstream's service protocol. - // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". + // Valid values are "http", "http2" and "grpc". Defaults to "http". Protocol string `mapstructure:"protocol"` // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. From 09d89945451fe6ac5c229cc94e4c251ece023d4c Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 2 Sep 2019 13:32:38 -0600 Subject: [PATCH 10/80] Only try to read CAFile if provided --- agent/structs/connect_proxy_config.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index d43edc4e280f..a40bc78ccf02 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -376,11 +376,13 @@ func (e *ExposeConfig) Finalize() error { } known[path.Path] = true - b, err := ioutil.ReadFile(path.CAFile) - if err != nil { - return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + if path.CAFile != "" { + b, err := ioutil.ReadFile(path.CAFile) + if err != nil { + return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + } + path.CACert = string(b) } - path.CACert = string(b) path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { From b29ed485716239da950267aa28f2611ae9581b16 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 2 Sep 2019 15:14:25 -0600 Subject: [PATCH 11/80] Fix deadlock in resetExposedChecks --- agent/agent.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index c5ca7a6fc4a0..608bff4d80c0 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3599,9 +3599,6 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPo // Future calls to check() will use the original target c.HTTP or c.GRPC // The agent stateLock MUST be held for this to be called func (a *Agent) resetExposedChecks(serviceID string) { - a.stateLock.Lock() - defer a.stateLock.Unlock() - for _, c := range a.checkHTTPs { if c.ServiceID == serviceID { c.ProxyHTTP = "" From 1f9d20263e9f6b4e5e85b3222040f63a3bd5677c Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 3 Sep 2019 20:49:16 -0600 Subject: [PATCH 12/80] Generate listener port for exposing checks --- agent/agent.go | 96 +++++++++++++++++++++++++++++++++------------ agent/agent_test.go | 10 +---- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 608bff4d80c0..56edcbae58ce 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -184,6 +184,9 @@ type Agent struct { // checkAliases maps the check ID to an associated Alias checks checkAliases map[types.CheckID]*checks.CheckAlias + // exposedPorts tracks listener ports for checks exposed through a proxy + exposedPorts map[string]int + // stateLock protects the agent state stateLock sync.Mutex @@ -2114,10 +2117,9 @@ func (a *Agent) addServiceInternal(service *structs.NodeService, chkTypes []*str } // If a proxy service wishes to expose checks, check targets need to be rerouted to the proxy listener - // This needs to be called after chkTypes are added to the agent, to avoid overwriting + // This needs to be called after chkTypes are added to the agent, to avoid being overwritten if service.Proxy.Expose.Checks { - err := a.rerouteExposedChecks( - service.Proxy.DestinationServiceID, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + err := a.rerouteExposedChecks(service.Proxy.DestinationServiceID, service.Proxy.LocalServiceAddress) if err != nil { a.logger.Println("[WARN] failed to reroute L7 checks to exposed proxy listener") } @@ -2430,11 +2432,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } if service.Proxy.Expose.Checks { - addr, err := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + port, err := a.listenerPort(service.ID, string(http.CheckID)) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into HTTP target") + a.logger.Printf("[ERR] agent: error exposing check: %s", err) + return err } + addr := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, port) http.ProxyHTTP = addr } @@ -2492,11 +2495,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } if service.Proxy.Expose.Checks { - addr, err := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + port, err := a.listenerPort(service.ID, string(grpc.CheckID)) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into GRPC target") + a.logger.Printf("[ERR] agent: error exposing check: %s", err) + return err } + addr := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, port) grpc.ProxyGRPC = addr } @@ -2655,6 +2659,11 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { } } + // Delete port from allocated port set + // If checks weren't being exposed then this is a no-op + portKey := fmt.Sprintf("%s:%s", svcID, checkID) + delete(a.exposedPorts, portKey) + a.cancelCheckMonitors(checkID) a.State.RemoveCheck(checkID) @@ -2666,6 +2675,7 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return err } } + a.logger.Printf("[DEBUG] agent: removed check %q", checkID) return nil } @@ -3569,12 +3579,16 @@ func (a *Agent) LocalState() *local.State { // rerouteExposedChecks will inject proxy address into check targets // Future calls to check() will dial the proxy listener // The agent stateLock MUST be held for this to be called -func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPort int) error { +func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string) error { for _, c := range a.checkHTTPs { if c.ServiceID != serviceID { continue } - addr, err := httpInjectAddr(c.HTTP, proxyAddr, proxyPort) + port, err := a.listenerPort(serviceID, string(c.CheckID)) + if err != nil { + return err + } + addr := httpInjectAddr(c.HTTP, proxyAddr, port) if err != nil { // The only way to get here is if the regex pattern fails to compile, which would be caught by tests return fmt.Errorf("failed to inject proxy addr into HTTP target") @@ -3585,11 +3599,11 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPo if c.ServiceID != serviceID { continue } - addr, err := grpcInjectAddr(c.GRPC, proxyAddr, proxyPort) + port, err := a.listenerPort(serviceID, string(c.CheckID)) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into GRPC target") + return err } + addr := grpcInjectAddr(c.GRPC, proxyAddr, port) c.ProxyGRPC = addr } return nil @@ -3609,31 +3623,61 @@ func (a *Agent) resetExposedChecks(serviceID string) { c.ProxyGRPC = "" } } + for k, _ := range a.exposedPorts { + if strings.HasPrefix(k, serviceID) { + delete(a.exposedPorts, k) + } + } +} + +// listenerPort allocates a port from the configured range +func (a *Agent) listenerPort(svcID, checkID string) (int, error) { + key := fmt.Sprintf("%s:%s", svcID, checkID) + if a.exposedPorts == nil { + a.exposedPorts = make(map[string]int) + } + if p, ok := a.exposedPorts[key]; ok { + return p, nil + } + + allocated := make(map[int]bool) + for _, v := range a.exposedPorts { + allocated[v] = true + } + + var port int + for i := a.config.ExposeMinPort; i < a.config.ExposeMaxPort; i++ { + port = a.config.ExposeMinPort + i + if !allocated[port] { + break + } + } + if port == 0 { + return 0, fmt.Errorf("no ports available to expose '%s'", checkID) + } + + return port, nil } // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] -func grpcInjectAddr(existing string, ip string, port int) (string, error) { +func grpcInjectAddr(existing string, ip string, port int) string { pattern := "(.*)((?::)(?:[0-9]+))(.*)$" - r, err := regexp.Compile(pattern) - if err != nil { - return "", err - } + r := regexp.MustCompile(pattern) + portRepl := fmt.Sprintf("${1}:%d${3}", port) out := r.ReplaceAllString(existing, portRepl) addrRepl := fmt.Sprintf("%s${2}${3}", ip) out = r.ReplaceAllString(out, addrRepl) - return out, nil + return out } // httpInjectAddr injects a port then an IP into a URL -func httpInjectAddr(url string, ip string, port int) (string, error) { +func httpInjectAddr(url string, ip string, port int) string { pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" - r, err := regexp.Compile(pattern) - if err != nil { - return "", err - } + r := regexp.MustCompile(pattern) + portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) out := r.ReplaceAllString(url, portRepl) @@ -3642,7 +3686,7 @@ func httpInjectAddr(url string, ip string, port int) (string, error) { addrRepl := fmt.Sprintf("${1}%s${3}${4}${5}", ip) out = r.ReplaceAllString(out, addrRepl) - return out, nil + return out } func fixIPv6(address string) string { diff --git a/agent/agent_test.go b/agent/agent_test.go index 34f702aaf3f6..998207019c74 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3439,10 +3439,7 @@ func TestAgent_grpcInjectAddr(t *testing.T) { } for _, tt := range tt { t.Run(tt.name, func(t *testing.T) { - got, err := grpcInjectAddr(tt.grpc, tt.ip, tt.port) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + got := grpcInjectAddr(tt.grpc, tt.ip, tt.port) if got != tt.want { t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) } @@ -3545,10 +3542,7 @@ func TestAgent_httpInjectAddr(t *testing.T) { } for _, tt := range tt { t.Run(tt.name, func(t *testing.T) { - got, err := httpInjectAddr(tt.url, tt.ip, tt.port) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + got := httpInjectAddr(tt.url, tt.ip, tt.port) if got != tt.want { t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) } From 4b632ff776c06a929881a78b2abed86de67b1117 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 3 Sep 2019 20:50:45 -0600 Subject: [PATCH 13/80] Update expose config to reflect listener per path --- agent/config/builder.go | 40 ++++++++++++++++++++++++- agent/config/config.go | 43 ++++++++++++++++++++++++++- agent/config/default.go | 2 ++ agent/config/runtime.go | 8 +++++ agent/structs/connect_proxy_config.go | 33 ++++++++++---------- 5 files changed, 107 insertions(+), 19 deletions(-) diff --git a/agent/config/builder.go b/agent/config/builder.go index 45db80843da3..731f3a72a335 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -22,7 +22,7 @@ import ( "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/consul/types" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-sockaddr/template" "golang.org/x/time/rate" ) @@ -369,6 +369,8 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { proxyMaxPort := b.portVal("ports.proxy_max_port", c.Ports.ProxyMaxPort) sidecarMinPort := b.portVal("ports.sidecar_min_port", c.Ports.SidecarMinPort) sidecarMaxPort := b.portVal("ports.sidecar_max_port", c.Ports.SidecarMaxPort) + exposeMinPort := b.portVal("ports.expose_min_port", c.Ports.ExposeMinPort) + exposeMaxPort := b.portVal("ports.expose_max_port", c.Ports.ExposeMaxPort) if proxyMaxPort < proxyMinPort { return RuntimeConfig{}, fmt.Errorf( "proxy_min_port must be less than proxy_max_port. To disable, set both to zero.") @@ -377,6 +379,10 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { return RuntimeConfig{}, fmt.Errorf( "sidecar_min_port must be less than sidecar_max_port. To disable, set both to zero.") } + if exposeMaxPort < exposeMinPort { + return RuntimeConfig{}, fmt.Errorf( + "expose_min_port must be less than expose_max_port. To disable, set both to zero.") + } // determine the default bind and advertise address // @@ -804,6 +810,8 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { ConnectCAConfig: connectCAConfig, ConnectSidecarMinPort: sidecarMinPort, ConnectSidecarMaxPort: sidecarMaxPort, + ExposeMinPort: exposeMinPort, + ExposeMaxPort: exposeMaxPort, DataDir: b.stringVal(c.DataDir), Datacenter: datacenter, DevMode: b.boolVal(b.Flags.DevMode), @@ -1305,6 +1313,7 @@ func (b *Builder) serviceProxyVal(v *ServiceProxy) *structs.ConnectProxyConfig { Config: v.Config, Upstreams: b.upstreamsVal(v.Upstreams), MeshGateway: b.meshGatewayConfVal(v.MeshGateway), + Expose: b.exposeConfVal(v.Expose), } } @@ -1345,6 +1354,35 @@ func (b *Builder) meshGatewayConfVal(mgConf *MeshGatewayConfig) structs.MeshGate return cfg } +func (b *Builder) exposeConfVal(v *ExposeConfig) structs.ExposeConfig { + var out structs.ExposeConfig + if v == nil { + return out + } + + out.Checks = b.boolVal(v.Checks) + out.Paths = b.pathsVal(v.Paths) + if err := out.Finalize(); err != nil { + b.err = multierror.Append(b.err, err) + } + return out +} + +func (b *Builder) pathsVal(v []Path) []structs.Path { + paths := make([]structs.Path, len(v)) + for i, p := range v { + paths[i] = structs.Path{ + ListenerPort: b.intVal(p.ListenerPort), + Path: b.stringVal(p.Path), + LocalPathPort: b.intVal(p.LocalPathPort), + Protocol: b.stringVal(p.Protocol), + TLSSkipVerify: b.boolVal(p.TLSSkipVerify), + CAFile: b.stringVal(p.CAFile), + } + } + return paths +} + func (b *Builder) serviceConnectVal(v *ServiceConnect) *structs.ServiceConnect { if v == nil { return nil diff --git a/agent/config/config.go b/agent/config/config.go index 50deea2cb878..e423f38079a1 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/hashicorp/consul/lib" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl" "github.com/mitchellh/mapstructure" ) @@ -97,6 +97,8 @@ func Parse(data string, format string) (c Config, err error) { "services.connect.sidecar_service.checks", "service.connect.sidecar_service.proxy.upstreams", "services.connect.sidecar_service.proxy.upstreams", + "service.connect.sidecar_service.proxy.expose.paths", + "services.connect.sidecar_service.proxy.expose.paths", }, []string{ "config_entries.bootstrap", // completely ignore this tree (fixed elsewhere) }) @@ -468,6 +470,9 @@ type ServiceProxy struct { // Mesh Gateway Configuration MeshGateway *MeshGatewayConfig `json:"mesh_gateway,omitempty" hcl:"mesh_gateway" mapstructure:"mesh_gateway"` + + // Expose defines whether checks or paths are exposed through the proxy + Expose *ExposeConfig `json:"expose,omitempty" hcl:"expose" mapstructure:"expose"` } // Upstream represents a single upstream dependency for a service or proxy. It @@ -513,6 +518,40 @@ type MeshGatewayConfig struct { Mode *string `json:"mode,omitempty" hcl:"mode" mapstructure:"mode"` } +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks *bool `json:"checks,omitempty" hcl:"checks" mapstructure:"checks"` + + // Port defines the port of the proxy's listener for exposed paths. + Port *int `json:"port,omitempty" hcl:"port" mapstructure:"port"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `json:"paths,omitempty" hcl:"paths" mapstructure:"paths"` +} + +type Path struct { + // ListenerPort defines the port of the proxy's listener for exposed paths. + ListenerPort *int `json:"listener_port,omitempty" hcl:"listener_port" mapstructure:"listener_port"` + + // Path is the path to expose through the proxy, ie. "/metrics." + Path *string `json:"path,omitempty" hcl:"path" mapstructure:"path"` + + // Protocol describes the upstream's service protocol. + Protocol *string `json:"protocol,omitempty" hcl:"protocol" mapstructure:"protocol"` + + // LocalPathPort is the port that the service is listening on for the given path. + LocalPathPort *int `json:"local_path_port,omitempty" hcl:"local_path_port" mapstructure:"local_path_port"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify *bool `json:"tls_skip_verify,omitempty" hcl:"tls_skip_verify" mapstructure:"tls_skip_verify"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile *string `json:"ca_file,omitempty" hcl:"ca_file" mapstructure:"ca_file"` +} + // AutoEncrypt is the agent-global auto_encrypt configuration. type AutoEncrypt struct { // TLS enables receiving certificates for clients from servers @@ -606,6 +645,8 @@ type Ports struct { ProxyMaxPort *int `json:"proxy_max_port,omitempty" hcl:"proxy_max_port" mapstructure:"proxy_max_port"` SidecarMinPort *int `json:"sidecar_min_port,omitempty" hcl:"sidecar_min_port" mapstructure:"sidecar_min_port"` SidecarMaxPort *int `json:"sidecar_max_port,omitempty" hcl:"sidecar_max_port" mapstructure:"sidecar_max_port"` + ExposeMinPort *int `json:"expose_min_port,omitempty" hcl:"expose_min_port" mapstructure:"expose_min_port"` + ExposeMaxPort *int `json:"expose_max_port,omitempty" hcl:"expose_max_port" mapstructure:"expose_max_port"` } type UnixSocket struct { diff --git a/agent/config/default.go b/agent/config/default.go index 1580e1915250..1ceeb94ad621 100644 --- a/agent/config/default.go +++ b/agent/config/default.go @@ -122,6 +122,8 @@ func DefaultSource() Source { proxy_max_port = 20255 sidecar_min_port = 21000 sidecar_max_port = 21255 + expose_min_port = 21500 + expose_max_port = 21755 } telemetry = { metrics_prefix = "consul" diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 2d2f9b0e1d72..1b21e3ff815a 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -541,6 +541,14 @@ type RuntimeConfig struct { // specified ConnectSidecarMaxPort int + // ExposeMinPort is the inclusive start of the range of ports + // allocated to the agent for exposing checks through a proxy + ExposeMinPort int + + // ExposeMinPort is the inclusive start of the range of ports + // allocated to the agent for exposing checks through a proxy + ExposeMaxPort int + // ConnectCAProvider is the type of CA provider to use with Connect. ConnectCAProvider string diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index a40bc78ccf02..a93220126023 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -11,11 +11,10 @@ import ( ) const ( - defaultExposePort = 21500 defaultExposeProtocol = "http" ) -var allowedExposeProtocols = map[string]bool{"http": true, "http2": true, "grpc": true} +var allowedExposeProtocols = map[string]bool{"http": true, "http2": true} type MeshGatewayMode string @@ -332,22 +331,22 @@ type ExposeConfig struct { // This flag triggers exposing all HTTP and GRPC check paths registered for the service. Checks bool `mapstructure:"checks"` - // Port defines the port of the proxy's listener for exposed paths. - Port int `mapstructure:"port"` - // Paths is the list of paths exposed through the proxy. Paths []Path `mapstructure:"paths"` } type Path struct { + // ListenerPort defines the port of the proxy's listener for exposed paths. + ListenerPort int `mapstructure:"listener_port"` + // Path is the path to expose through the proxy, ie. "/metrics." Path string `mapstructure:"path"` - // Port is the port that the service is listening on for the given path. - Port int `mapstructure:"port"` + // LocalPathPort is the port that the service is listening on for the given path. + LocalPathPort int `mapstructure:"local_path_port"` // Protocol describes the upstream's service protocol. - // Valid values are "http", "http2" and "grpc". Defaults to "http". + // Valid values are "http" and "http2", defaults to "http" Protocol string `mapstructure:"protocol"` // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. @@ -362,20 +361,19 @@ type Path struct { // Finalize validates ExposeConfig and sets default values func (e *ExposeConfig) Finalize() error { - if e.Port < 0 || e.Port > 65535 { - return fmt.Errorf("invalid port: %d", e.Port) - } - if e.Port == 0 { - e.Port = defaultExposePort - } - var known = make(map[string]bool) - for _, path := range e.Paths { + for i := 0; i < len(e.Paths); i++ { + path := &e.Paths[i] + if seen := known[path.Path]; seen { return fmt.Errorf("duplicate paths exposed") } known[path.Path] = true + if path.ListenerPort <= 0 || path.ListenerPort > 65535 { + return fmt.Errorf("invalid port: %d", path.ListenerPort) + } + if path.CAFile != "" { b, err := ioutil.ReadFile(path.CAFile) if err != nil { @@ -386,7 +384,8 @@ func (e *ExposeConfig) Finalize() error { path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { - return fmt.Errorf("protocol '%s' not recognized for path: %s", path.Protocol, path.Path) + return fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", + path.Protocol, path.Path) } if path.Protocol == "" { path.Protocol = defaultExposeProtocol From 02baf1365f09354267a343520f944165b1f6f842 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 3 Sep 2019 20:56:54 -0600 Subject: [PATCH 14/80] Create Envoy listeners and clusters for exposed paths --- agent/xds/clusters.go | 31 +++++++--- agent/xds/listeners.go | 131 +++++++++++++++++++++++++++++++---------- 2 files changed, 125 insertions(+), 37 deletions(-) diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 69d3be65a8e4..a354a5d11ee8 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -44,7 +44,7 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh clusters := make([]proto.Message, 0, len(cfgSnap.Proxy.Upstreams)+1) // Include the "app" cluster for the public listener - appCluster, err := s.makeAppCluster(cfgSnap) + appCluster, err := s.makeAppCluster(cfgSnap, LocalAppClusterName, "", cfgSnap.Proxy.LocalServicePort) if err != nil { return nil, err } @@ -74,9 +74,26 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } } + // Create a new cluster if we need to expose a port that is different from the service port + for _, path := range cfgSnap.Proxy.Expose.Paths { + if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { + continue + } + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path), path.Protocol, path.LocalPathPort) + if err != nil { + s.Logger.Printf("[WARN] envoy: failed to make local cluster for '%s': %s", path.Path, err) + continue + } + clusters = append(clusters, c) + } + return clusters, nil } +func makeExposeClusterName(path structs.Path) string { + return fmt.Sprintf("exposed_cluster_%d", path.LocalPathPort) +} + // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters" // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. @@ -122,7 +139,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho return clusters, nil } -func (s *Server) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot) (*envoy.Cluster, error) { +func (s *Server) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathProtocol string, port int) (*envoy.Cluster, error) { var c *envoy.Cluster var err error @@ -143,23 +160,23 @@ func (s *Server) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot) (*envoy.Cluste addr = "127.0.0.1" } c = &envoy.Cluster{ - Name: LocalAppClusterName, + Name: name, ConnectTimeout: time.Duration(cfg.LocalConnectTimeoutMs) * time.Millisecond, ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_STATIC}, LoadAssignment: &envoy.ClusterLoadAssignment{ - ClusterName: LocalAppClusterName, + ClusterName: name, Endpoints: []envoyendpoint.LocalityLbEndpoints{ { LbEndpoints: []envoyendpoint.LbEndpoint{ - makeEndpoint(LocalAppClusterName, + makeEndpoint(name, addr, - cfgSnap.Proxy.LocalServicePort), + port), }, }, }, }, } - if cfg.Protocol == "http2" || cfg.Protocol == "grpc" { + if cfg.Protocol == "http2" || cfg.Protocol == "grpc" || pathProtocol == "http2" { c.Http2ProtocolOptions = &envoycore.Http2ProtocolOptions{} } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 9c635e1056ec..dc5e9e787503 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "strings" envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" @@ -71,6 +72,16 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps } resources[i+1] = upstreamListener } + + // Configure additional listener for exposed check paths + for _, path := range cfgSnap.Proxy.Expose.Paths { + l, err := s.makeExposedCheckListener(cfgSnap, path) + if err != nil { + return nil, err + } + resources = append(resources, l) + } + return resources, nil } @@ -253,7 +264,8 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri l = makeListener(PublicListenerName, addr, port) - filter, err := makeListenerFilter(false, cfg.Protocol, "public_listener", LocalAppClusterName, "", true) + filter, err := makeListenerFilter( + false, cfg.Protocol, "public_listener", LocalAppClusterName, "", "", true) if err != nil { return nil, err } @@ -270,6 +282,49 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri return l, err } +func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, path structs.Path) (proto.Message, error) { + cfg, err := ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + } + + // No user config, use default listener + addr := cfgSnap.Address + + // Override with bind address if one is set, otherwise default to 0.0.0.0 + if cfg.BindAddress != "" { + addr = cfg.BindAddress + } else if addr == "" { + addr = "0.0.0.0" + } + + // Strip any special characters from path + r := regexp.MustCompile(`[^a-zA-Z0-9]+`) + strippedPath := r.ReplaceAllString(path.Path, "") + listenerName := fmt.Sprintf("exposed_path_listener_%s_%d", strippedPath, path.ListenerPort) + l := makeListener(listenerName, addr, path.ListenerPort) + + filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, path.ListenerPort) + clusterName := LocalAppClusterName + if path.LocalPathPort != cfgSnap.Proxy.LocalServicePort { + clusterName = makeExposeClusterName(path) + } + + f, err := makeListenerFilter(false, path.Protocol, filterName, clusterName, "", path.Path, true) + if err != nil { + return nil, err + } + + l.FilterChains = []envoylistener.FilterChain{ + { + Filters: []envoylistener.Filter{f}, + }, + } + return l, err +} + // makeUpstreamListenerIgnoreDiscoveryChain counterintuitively takes an (optional) chain func (s *Server) makeUpstreamListenerIgnoreDiscoveryChain( u *structs.Upstream, @@ -303,7 +358,8 @@ func (s *Server) makeUpstreamListenerIgnoreDiscoveryChain( clusterName := CustomizeClusterName(sni, chain) l := makeListener(upstreamID, addr, u.LocalBindPort) - filter, err := makeListenerFilter(false, cfg.Protocol, upstreamID, clusterName, "upstream_", false) + filter, err := makeListenerFilter( + false, cfg.Protocol, upstreamID, clusterName, "upstream_", "", false) if err != nil { return nil, err } @@ -408,7 +464,8 @@ func (s *Server) makeUpstreamListenerForDiscoveryChain( proto = "tcp" } - filter, err := makeListenerFilter(true, proto, upstreamID, "", "upstream_", false) + filter, err := makeListenerFilter( + true, proto, upstreamID, "", "upstream_", "", false) if err != nil { return nil, err } @@ -423,14 +480,17 @@ func (s *Server) makeUpstreamListenerForDiscoveryChain( return l, nil } -func makeListenerFilter(useRDS bool, protocol, filterName, cluster, statPrefix string, ingress bool) (envoylistener.Filter, error) { +func makeListenerFilter( + useRDS bool, + protocol, filterName, cluster, statPrefix, routePath string, ingress bool) (envoylistener.Filter, error) { + switch protocol { case "grpc": - return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, ingress, true, true) + return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, routePath, ingress, true, true) case "http2": - return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, ingress, false, true) + return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, routePath, ingress, false, true) case "http": - return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, ingress, false, false) + return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, routePath, ingress, false, false) case "tcp": fallthrough default: @@ -471,7 +531,7 @@ func makeStatPrefix(protocol, prefix, filterName string) string { func makeHTTPFilter( useRDS bool, - filterName, cluster, statPrefix string, + filterName, cluster, statPrefix, routePath string, ingress, grpc, http2 bool, ) (envoylistener.Filter, error) { op := envoyhttp.INGRESS @@ -482,9 +542,14 @@ func makeHTTPFilter( if grpc { proto = "grpc" } + codec := envoyhttp.AUTO + if grpc || http2 { + codec = envoyhttp.HTTP2 + } + cfg := &envoyhttp.HttpConnectionManager{ StatPrefix: makeStatPrefix(proto, statPrefix, filterName), - CodecType: envoyhttp.AUTO, + CodecType: codec, HttpFilters: []*envoyhttp.HttpFilter{ &envoyhttp.HttpFilter{ Name: "envoy.router", @@ -517,33 +582,39 @@ func makeHTTPFilter( if cluster == "" { return envoylistener.Filter{}, fmt.Errorf("must specify cluster name when not using RDS") } + route := envoyroute.Route{ + Match: envoyroute.RouteMatch{ + PathSpecifier: &envoyroute.RouteMatch_Prefix{ + Prefix: "/", + }, + // TODO(banks) Envoy supports matching only valid GRPC + // requests which might be nice to add here for gRPC services + // but it's not supported in our current envoy SDK version + // although docs say it was supported by 1.8.0. Going to defer + // that until we've updated the deps. + }, + Action: &envoyroute.Route_Route{ + Route: &envoyroute.RouteAction{ + ClusterSpecifier: &envoyroute.RouteAction_Cluster{ + Cluster: cluster, + }, + }, + }, + } + // If a path is provided, do not match on a catch-all prefix + if routePath != "" { + route.Match.PathSpecifier = &envoyroute.RouteMatch_Path{Path: routePath} + } + cfg.RouteSpecifier = &envoyhttp.HttpConnectionManager_RouteConfig{ RouteConfig: &envoy.RouteConfiguration{ Name: filterName, VirtualHosts: []envoyroute.VirtualHost{ - envoyroute.VirtualHost{ + { Name: filterName, Domains: []string{"*"}, Routes: []envoyroute.Route{ - envoyroute.Route{ - Match: envoyroute.RouteMatch{ - PathSpecifier: &envoyroute.RouteMatch_Prefix{ - Prefix: "/", - }, - // TODO(banks) Envoy supports matching only valid GRPC - // requests which might be nice to add here for gRPC services - // but it's not supported in our current envoy SDK version - // although docs say it was supported by 1.8.0. Going to defer - // that until we've updated the deps. - }, - Action: &envoyroute.Route_Route{ - Route: &envoyroute.RouteAction{ - ClusterSpecifier: &envoyroute.RouteAction_Cluster{ - Cluster: cluster, - }, - }, - }, - }, + route, }, }, }, @@ -557,7 +628,7 @@ func makeHTTPFilter( if grpc { // Add grpc bridge before router - cfg.HttpFilters = append([]*envoyhttp.HttpFilter{&envoyhttp.HttpFilter{ + cfg.HttpFilters = append([]*envoyhttp.HttpFilter{{ Name: "envoy.grpc_http1_bridge", ConfigType: &envoyhttp.HttpFilter_Config{Config: &types.Struct{}}, }}, cfg.HttpFilters...) From c999f0d9ca5d4ff0e5a42829d1af7a2971ac204c Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 14:04:10 -0600 Subject: [PATCH 15/80] Update configuration and add cfg tests --- agent/config/config.go | 4 + agent/config/runtime_test.go | 141 +++++++++++++++++- agent/structs/check_type.go | 4 + agent/structs/connect_proxy_config.go | 4 +- agent/structs/connect_proxy_config_test.go | 158 +++++++++++++++++++++ 5 files changed, 306 insertions(+), 5 deletions(-) diff --git a/agent/config/config.go b/agent/config/config.go index e423f38079a1..4f4d53a71db0 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -89,8 +89,12 @@ func Parse(data string, format string) (c Config, err error) { "services.connect.proxy.config.upstreams", // Deprecated "service.connect.proxy.upstreams", "services.connect.proxy.upstreams", + "service.connect.proxy.expose.paths", + "services.connect.proxy.expose.paths", "service.proxy.upstreams", "services.proxy.upstreams", + "service.proxy.expose.paths", + "services.proxy.expose.paths", // Need all the service(s) exceptions also for nested sidecar service. "service.connect.sidecar_service.checks", diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index a114132f91a3..ba7526c4af26 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -1295,6 +1295,36 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { rt.DataDir = dataDir }, }, + { + desc: "min/max ports for dynamic exposed listeners", + args: []string{`-data-dir=` + dataDir}, + json: []string{`{ + "ports": { + "expose_min_port": 1234, + "expose_max_port": 5678 + } + }`}, + hcl: []string{` + ports { + expose_min_port = 1234 + expose_max_port = 5678 + } + `}, + patch: func(rt *RuntimeConfig) { + rt.ExposeMinPort = 1234 + rt.ExposeMaxPort = 5678 + rt.DataDir = dataDir + }, + }, + { + desc: "defaults for dynamic exposed listeners", + args: []string{`-data-dir=` + dataDir}, + patch: func(rt *RuntimeConfig) { + rt.ExposeMinPort = 21500 + rt.ExposeMaxPort = 21755 + rt.DataDir = dataDir + }, + }, // ------------------------------------------------------------ // precedence rules @@ -2338,6 +2368,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ], "proxy": { + "expose": { + "checks": true, + "paths": [ + { + "path": "/health", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http" + } + ] + }, "upstreams": [ { "destination_name": "db", @@ -2363,6 +2404,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ] proxy { + expose { + checks = true + paths = [ + { + path = "/health" + local_path_port = 8080 + listener_port = 21500 + protocol = "http" + } + ] + }, upstreams = [ { destination_name = "db" @@ -2391,6 +2443,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { }, }, Proxy: &structs.ConnectProxyConfig{ + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.Path{ + { + Path: "/health", + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http", + }, + }, + }, Upstreams: structs.Upstreams{ structs.Upstream{ DestinationType: "service", @@ -2434,6 +2497,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ], "proxy": { + "expose": { + "checks": true, + "paths": [ + { + "path": "/health", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http" + } + ] + }, "upstreams": [ { "destination_name": "db", @@ -2459,6 +2533,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ] proxy { + expose { + checks = true + paths = [ + { + path = "/health" + local_path_port = 8080 + listener_port = 21500 + protocol = "http" + } + ] + }, upstreams = [ { destination_name = "db" @@ -2487,6 +2572,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { }, }, Proxy: &structs.ConnectProxyConfig{ + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.Path{ + { + Path: "/health", + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http", + }, + }, + }, Upstreams: structs.Upstreams{ structs.Upstream{ DestinationType: "service", @@ -3627,7 +3723,9 @@ func TestFullConfig(t *testing.T) { "server": 3757, "grpc": 4881, "sidecar_min_port": 8888, - "sidecar_max_port": 9999 + "sidecar_max_port": 9999, + "expose_min_port": 1111, + "expose_max_port": 2222 }, "protocol": 30793, "primary_datacenter": "ejtmd43d", @@ -3871,6 +3969,17 @@ func TestFullConfig(t *testing.T) { "destination_service_name": "6L6BVfgH", "local_service_address": "127.0.0.2", "local_service_port": 23759, + "expose": { + "checks": true, + "paths": [ + { + "path": "/health", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http" + } + ] + }, "upstreams": [ { "destination_name": "KPtAj2cb", @@ -4205,8 +4314,8 @@ func TestFullConfig(t *testing.T) { } pid_file = "43xN80Km" ports { - dns = 7001, - http = 7999, + dns = 7001 + http = 7999 https = 15127 server = 3757 grpc = 4881 @@ -4214,6 +4323,8 @@ func TestFullConfig(t *testing.T) { proxy_max_port = 3000 sidecar_min_port = 8888 sidecar_max_port = 9999 + expose_min_port = 1111 + expose_max_port = 2222 } protocol = 30793 primary_datacenter = "ejtmd43d" @@ -4472,6 +4583,17 @@ func TestFullConfig(t *testing.T) { local_bind_address = "127.24.88.0" }, ] + expose { + checks = true + paths = [ + { + path = "/health" + local_path_port = 8080 + listener_port = 21500 + protocol = "http" + } + ] + } } }, { @@ -4797,6 +4919,8 @@ func TestFullConfig(t *testing.T) { ConnectEnabled: true, ConnectSidecarMinPort: 8888, ConnectSidecarMaxPort: 9999, + ExposeMinPort: 1111, + ExposeMaxPort: 2222, ConnectCAProvider: "consul", ConnectCAConfig: map[string]interface{}{ "RotationPeriod": "90h", @@ -5044,6 +5168,17 @@ func TestFullConfig(t *testing.T) { LocalBindAddress: "127.24.88.0", }, }, + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.Path{ + { + Path: "/health", + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http", + }, + }, + }, }, Weights: &structs.Weights{ Passing: 1, diff --git a/agent/structs/check_type.go b/agent/structs/check_type.go index 9b1b055dae30..1b217e2e91ab 100644 --- a/agent/structs/check_type.go +++ b/agent/structs/check_type.go @@ -41,6 +41,10 @@ type CheckType struct { Timeout time.Duration TTL time.Duration + // Definition fields used when exposing checks through a proxy + ProxyHTTP string + ProxyGRPC string + // DeregisterCriticalServiceAfter, if >0, will cause the associated // service, if any, to be deregistered if this check is critical for // longer than this duration. diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index a93220126023..db9260070c10 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -371,13 +371,13 @@ func (e *ExposeConfig) Finalize() error { known[path.Path] = true if path.ListenerPort <= 0 || path.ListenerPort > 65535 { - return fmt.Errorf("invalid port: %d", path.ListenerPort) + return fmt.Errorf("invalid listener port: %d", path.ListenerPort) } if path.CAFile != "" { b, err := ioutil.ReadFile(path.CAFile) if err != nil { - return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + return fmt.Errorf("failed to read CAFile '%s': %v", path.CAFile, err) } path.CACert = string(b) } diff --git a/agent/structs/connect_proxy_config_test.go b/agent/structs/connect_proxy_config_test.go index bc1dd0f50d17..6326ab253c33 100644 --- a/agent/structs/connect_proxy_config_test.go +++ b/agent/structs/connect_proxy_config_test.go @@ -3,6 +3,9 @@ package structs import ( "encoding/json" "fmt" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" "testing" "github.com/hashicorp/consul/api" @@ -270,3 +273,158 @@ func TestValidateMeshGatewayMode(t *testing.T) { }) } } + +func TestExposeConfig_Finalize(t *testing.T) { + t.Parallel() + + tmpDir, err := ioutil.TempDir("", "exposeconfig_") + if err != nil { + t.Fatalf("failed to create tempdir: %v", err) + } + defer os.RemoveAll(tmpDir) + + tmpFile, err := ioutil.TempFile(tmpDir, "CAFile") + if err != nil { + t.Fatalf("failed to create tempfile: %v", err) + } + defer tmpFile.Close() + + type fields struct { + Checks bool + Paths []Path + } + tests := []struct { + name string + fields fields + want ExposeConfig + wantErr bool + errMsg string + }{ + { + name: "duplicate path", + fields: fields{ + Paths: []Path{ + { + LocalPathPort: 80, + ListenerPort: 80, + Path: "/metrics", + }, + { + LocalPathPort: 80, + ListenerPort: 80, + Path: "/metrics", + }, + }, + }, + wantErr: true, + errMsg: "duplicate paths exposed", + }, + { + name: "negative listener port", + fields: fields{ + Paths: []Path{ + { + ListenerPort: -1, + }, + }, + }, + wantErr: true, + errMsg: "invalid listener port: -1", + }, + { + name: "listener port too large", + fields: fields{ + Paths: []Path{ + { + ListenerPort: 65536, + }, + }, + }, + wantErr: true, + errMsg: "invalid listener port: 65536", + }, + { + name: "protocol not supported", + fields: fields{ + Paths: []Path{ + { + Path: "/metrics", + LocalPathPort: 80, + ListenerPort: 80, + Protocol: "tcp", + }, + }, + }, + wantErr: true, + errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", + }, + { + name: "protocol not supported", + fields: fields{ + Paths: []Path{ + { + Path: "/metrics", + LocalPathPort: 80, + ListenerPort: 80, + Protocol: "tcp", + }, + }, + }, + wantErr: true, + errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", + }, + { + name: "default to http when no protocol", + fields: fields{ + Paths: []Path{ + { + LocalPathPort: 80, + ListenerPort: 80, + }, + }, + }, + wantErr: false, + want: ExposeConfig{ + Paths: []Path{ + {LocalPathPort: 80, ListenerPort: 80, Protocol: "http"}, + }, + }, + }, + { + name: "lowercase protocol", + fields: fields{ + Paths: []Path{ + { + LocalPathPort: 80, + ListenerPort: 80, + Protocol: "HTTP2", + }, + }, + }, + wantErr: false, + want: ExposeConfig{ + Paths: []Path{ + {LocalPathPort: 80, ListenerPort: 80, Protocol: "http2"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &ExposeConfig{ + Checks: tt.fields.Checks, + Paths: tt.fields.Paths, + } + err := e.Finalize() + if (err != nil) != tt.wantErr { + t.Errorf("Finalize() got error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr && tt.errMsg != err.Error() { + t.Errorf("Finalize() got error: '%v', want: '%s'", err, tt.errMsg) + } + if !tt.wantErr { + assert.Equal(t, &tt.want, e) + } + }) + } +} From 949c081c483760c7145f896295a3ec95ad004909 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 14:04:50 -0600 Subject: [PATCH 16/80] Create clusters and listeners for expose Consul checks --- agent/agent.go | 11 ++--- agent/checks/check.go | 10 +++-- agent/xds/clusters.go | 23 +++++++--- agent/xds/listeners.go | 98 +++++++++++++++++++++++++++++++++++------- agent/xds/server.go | 7 +++ 5 files changed, 120 insertions(+), 29 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 56edcbae58ce..2171feda99e1 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -670,6 +670,7 @@ func (a *Agent) listenAndServeGRPC() error { CfgMgr: a.proxyConfig, Authz: a, ResolveToken: a.resolveToken, + CheckFetcher: a, } a.xdsServer.Initialize() @@ -3631,6 +3632,7 @@ func (a *Agent) resetExposedChecks(serviceID string) { } // listenerPort allocates a port from the configured range +// The agent stateLock MUST be held when this is called func (a *Agent) listenerPort(svcID, checkID string) (int, error) { key := fmt.Sprintf("%s:%s", svcID, checkID) if a.exposedPorts == nil { @@ -3646,9 +3648,10 @@ func (a *Agent) listenerPort(svcID, checkID string) (int, error) { } var port int - for i := a.config.ExposeMinPort; i < a.config.ExposeMaxPort; i++ { + for i := 0; i < a.config.ExposeMaxPort-a.config.ExposeMinPort; i++ { port = a.config.ExposeMinPort + i if !allocated[port] { + a.exposedPorts[key] = port break } } @@ -3661,8 +3664,7 @@ func (a *Agent) listenerPort(svcID, checkID string) (int, error) { // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] func grpcInjectAddr(existing string, ip string, port int) string { - pattern := "(.*)((?::)(?:[0-9]+))(.*)$" - r := regexp.MustCompile(pattern) + r := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") portRepl := fmt.Sprintf("${1}:%d${3}", port) out := r.ReplaceAllString(existing, portRepl) @@ -3675,8 +3677,7 @@ func grpcInjectAddr(existing string, ip string, port int) string { // httpInjectAddr injects a port then an IP into a URL func httpInjectAddr(url string, ip string, port int) string { - pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" - r := regexp.MustCompile(pattern) + r := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) out := r.ReplaceAllString(url, portRepl) diff --git a/agent/checks/check.go b/agent/checks/check.go index d25566ff7ae2..bb4ca538e99a 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -338,6 +338,7 @@ func (c *CheckHTTP) CheckType() structs.CheckType { Method: c.Method, Header: c.Header, Interval: c.Interval, + ProxyHTTP: c.ProxyHTTP, Timeout: c.Timeout, OutputMaxSize: c.OutputMaxSize, } @@ -702,10 +703,11 @@ type CheckGRPC struct { func (c *CheckGRPC) CheckType() structs.CheckType { return structs.CheckType{ - CheckID: c.CheckID, - GRPC: c.GRPC, - Interval: c.Interval, - Timeout: c.Timeout, + CheckID: c.CheckID, + GRPC: c.GRPC, + ProxyGRPC: c.ProxyGRPC, + Interval: c.Interval, + Timeout: c.Timeout, } } diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index a354a5d11ee8..70b2fa87a5fb 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -74,24 +74,37 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } } + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create clusters for if needed + if cfgSnap.Proxy.Expose.Checks { + for _, check := range s.CheckFetcher.ServiceHTTPChecks(cfgSnap.Proxy.DestinationServiceID) { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Printf("[WARN] envoy: failed to create cluster for check '%s': %v", check.CheckID, err) + continue + } + paths = append(paths, p) + } + } + // Create a new cluster if we need to expose a port that is different from the service port - for _, path := range cfgSnap.Proxy.Expose.Paths { + for _, path := range paths { if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { continue } - c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path), path.Protocol, path.LocalPathPort) + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path.LocalPathPort), path.Protocol, path.LocalPathPort) if err != nil { s.Logger.Printf("[WARN] envoy: failed to make local cluster for '%s': %s", path.Path, err) continue } clusters = append(clusters, c) } - return clusters, nil } -func makeExposeClusterName(path structs.Path) string { - return fmt.Sprintf("exposed_cluster_%d", path.LocalPathPort) +func makeExposeClusterName(destinationPort int) string { + return fmt.Sprintf("exposed_cluster_%d", destinationPort) } // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters" diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index dc5e9e787503..28269b5216c0 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "regexp" + "strconv" "strings" envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" @@ -73,9 +74,28 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps resources[i+1] = upstreamListener } + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create listeners for if needed + if cfgSnap.Proxy.Expose.Checks { + for _, check := range s.CheckFetcher.ServiceHTTPChecks(cfgSnap.Proxy.DestinationServiceID) { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Printf("[WARN] envoy: failed to create listener for check '%s': %v", check.CheckID, err) + continue + } + paths = append(paths, p) + } + } + // Configure additional listener for exposed check paths - for _, path := range cfgSnap.Proxy.Expose.Paths { - l, err := s.makeExposedCheckListener(cfgSnap, path) + for _, path := range paths { + clusterName := LocalAppClusterName + if path.LocalPathPort != cfgSnap.Proxy.LocalServicePort { + clusterName = makeExposeClusterName(path.LocalPathPort) + } + + l, err := s.makeExposedCheckListener(cfgSnap, clusterName, path.Path, path.Protocol, path.ListenerPort) if err != nil { return nil, err } @@ -85,13 +105,64 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps return resources, nil } +func parseCheckPath(check structs.CheckType) (structs.Path, error) { + grpcRE := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") + httpRE := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) + + var path structs.Path + var err error + + if check.HTTP != "" { + path.Protocol = "http" + + matches := httpRE.FindStringSubmatch(check.HTTP) + path.Path = matches[4] + + localStr := strings.TrimPrefix(matches[3], ":") + path.LocalPathPort, err = strconv.Atoi(localStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.HTTP, err) + } + + matches = httpRE.FindStringSubmatch(check.ProxyHTTP) + + listenerStr := strings.TrimPrefix(matches[3], ":") + path.ListenerPort, err = strconv.Atoi(listenerStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyHTTP, err) + } + } + + if check.GRPC != "" { + path.Path = "/grpc.health.v1.Health/Check" + path.Protocol = "http2" + + matches := grpcRE.FindStringSubmatch(check.GRPC) + + localStr := strings.TrimPrefix(matches[2], ":") + path.LocalPathPort, err = strconv.Atoi(localStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.GRPC, err) + } + + matches = grpcRE.FindStringSubmatch(check.ProxyGRPC) + + listenerStr := strings.TrimPrefix(matches[2], ":") + path.ListenerPort, err = strconv.Atoi(listenerStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyGRPC, err) + } + } + return path, nil +} + // listenersFromSnapshotMeshGateway returns the "listener" for a mesh-gateway service func (s *Server) listenersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) { cfg, err := ParseMeshGatewayConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. - s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %v", err) } // TODO - prevent invalid configurations of binding to the same port/addr @@ -232,7 +303,7 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. - s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %v", err) } if cfg.PublicListenerJSON != "" { @@ -282,12 +353,12 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri return l, err } -func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, path structs.Path) (proto.Message, error) { +func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, cluster, path, protocol string, port int) (proto.Message, error) { cfg, err := ParseProxyConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. - s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %v", err) } // No user config, use default listener @@ -302,17 +373,14 @@ func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, path // Strip any special characters from path r := regexp.MustCompile(`[^a-zA-Z0-9]+`) - strippedPath := r.ReplaceAllString(path.Path, "") - listenerName := fmt.Sprintf("exposed_path_listener_%s_%d", strippedPath, path.ListenerPort) - l := makeListener(listenerName, addr, path.ListenerPort) + strippedPath := r.ReplaceAllString(path, "") + listenerName := fmt.Sprintf("exposed_path_%s", strippedPath) - filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, path.ListenerPort) - clusterName := LocalAppClusterName - if path.LocalPathPort != cfgSnap.Proxy.LocalServicePort { - clusterName = makeExposeClusterName(path) - } + l := makeListener(listenerName, addr, port) + + filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, port) - f, err := makeListenerFilter(false, path.Protocol, filterName, clusterName, "", path.Path, true) + f, err := makeListenerFilter(false, protocol, filterName, cluster, "", path, true) if err != nil { return nil, err } diff --git a/agent/xds/server.go b/agent/xds/server.go index f4ee9fa15c7b..1cc346dba044 100644 --- a/agent/xds/server.go +++ b/agent/xds/server.go @@ -97,6 +97,12 @@ type ConnectAuthz interface { ConnectAuthorize(token string, req *structs.ConnectAuthorizeRequest) (authz bool, reason string, m *cache.ResultMeta, err error) } +// ServiceChecks is the interface the agent needs to expose +// for the xDS server to fetch a service's HTTP check definitions +type HTTPCheckFetcher interface { + ServiceHTTPChecks(serviceID string) []structs.CheckType +} + // ConfigManager is the interface xds.Server requires to consume proxy config // updates. It's satisfied normally by the agent's proxycfg.Manager, but allows // easier testing without several layers of mocked cache, local state and @@ -121,6 +127,7 @@ type Server struct { // This is only used during idle periods of stream interactions (i.e. when // there has been no recent DiscoveryRequest). AuthCheckFrequency time.Duration + CheckFetcher HTTPCheckFetcher } // Initialize will finish configuring the Server for first use. From 6e9892e4888736cf00cbe771119804e1b3079e30 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 14:14:20 -0600 Subject: [PATCH 17/80] Fix grpc test --- agent/checks/grpc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/grpc_test.go b/agent/checks/grpc_test.go index 3c86093b0e70..ebfe8aa518dc 100644 --- a/agent/checks/grpc_test.go +++ b/agent/checks/grpc_test.go @@ -88,7 +88,7 @@ func TestCheck(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { probe := NewGrpcHealthProbe(tt.args.target, tt.args.timeout, tt.args.tlsConfig) - actualError := probe.Check() + actualError := probe.Check(tt.args.target) actuallyHealthy := actualError == nil if tt.healthy != actuallyHealthy { t.Errorf("FAIL: %s. Expected healthy %t, but err == %v", tt.name, tt.healthy, actualError) From 5b4f10539f18c7faaa2e731dec2549859d658e73 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 16:00:15 -0600 Subject: [PATCH 18/80] Fix broken tests --- agent/agent_endpoint.go | 7 ++- agent/config/runtime_test.go | 4 ++ agent/structs/service_definition.go | 1 + agent/structs/structs.go | 2 +- agent/structs/structs_filtering_test.go | 55 +++++++++++++++++++ ...nect-proxy-with-chain-and-overrides.golden | 1 + .../connect-proxy-with-grpc-chain.golden | 1 + .../connect-proxy-with-http2-chain.golden | 1 + api/agent.go | 1 + api/config_entry.go | 35 ++++++++++++ 10 files changed, 106 insertions(+), 2 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index c7a887e465b7..78a8df6a69d2 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -24,7 +24,7 @@ import ( "github.com/hashicorp/consul/lib/file" "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/types" - bexpr "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/go-bexpr" "github.com/hashicorp/logutils" "github.com/hashicorp/serf/coordinate" "github.com/hashicorp/serf/serf" @@ -769,6 +769,11 @@ func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Re "local_service_address": "LocalServiceAddress", // SidecarService "sidecar_service": "SidecarService", + // Expose Config + "local_path_port": "LocalPathPort", + "listener_port": "ListenerPort", + "tls_skip_verify": "TLSSkipVerify", + "ca_file": "CAFile", // DON'T Recurse into these opaque config maps or we might mangle user's // keys. Note empty canonical is a special sentinel to prevent recursion. diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index ba7526c4af26..60b6a1a62d30 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -5818,6 +5818,8 @@ func TestSanitize(t *testing.T) { "EncryptKey": "hidden", "EncryptVerifyIncoming": false, "EncryptVerifyOutgoing": false, + "ExposeMaxPort": 0, + "ExposeMinPort": 0, "GRPCAddrs": [], "GRPCPort": 0, "HTTPAddrs": [ @@ -5898,6 +5900,8 @@ func TestSanitize(t *testing.T) { "Name": "blurb", "Notes": "", "OutputMaxSize": ` + strconv.Itoa(checks.DefaultBufSize) + `, + "ProxyGRPC": "", + "ProxyHTTP": "", "ScriptArgs": [], "Shell": "", "Status": "", diff --git a/agent/structs/service_definition.go b/agent/structs/service_definition.go index 26b296887d2d..0ba6c6c5eb9f 100644 --- a/agent/structs/service_definition.go +++ b/agent/structs/service_definition.go @@ -54,6 +54,7 @@ func (s *ServiceDefinition) NodeService() *NodeService { ns.Proxy.Upstreams[i].DestinationType = UpstreamDestTypeService } } + ns.Proxy.Expose = s.Proxy.Expose } if ns.ID == "" && ns.Service != "" { ns.ID = ns.Service diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 1676b85e7085..7880fef0b486 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -14,7 +14,7 @@ import ( "time" "github.com/hashicorp/go-msgpack/codec" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/serf/coordinate" "github.com/mitchellh/hashstructure" diff --git a/agent/structs/structs_filtering_test.go b/agent/structs/structs_filtering_test.go index ffd9342f209b..6b3ae2091387 100644 --- a/agent/structs/structs_filtering_test.go +++ b/agent/structs/structs_filtering_test.go @@ -60,6 +60,57 @@ var expectedFieldConfigMeshGatewayConfig bexpr.FieldConfigurations = bexpr.Field }, } +var expectedFieldConfigExposeConfig bexpr.FieldConfigurations = bexpr.FieldConfigurations{ + "Checks": &bexpr.FieldConfiguration{ + StructFieldName: "Checks", + CoerceFn: bexpr.CoerceBool, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Paths": &bexpr.FieldConfiguration{ + StructFieldName: "Paths", + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchIsEmpty, bexpr.MatchIsNotEmpty}, + SubFields: expectedFieldConfigPaths, + }, +} + +var expectedFieldConfigPaths bexpr.FieldConfigurations = bexpr.FieldConfigurations{ + "ListenerPort": &bexpr.FieldConfiguration{ + StructFieldName: "ListenerPort", + CoerceFn: bexpr.CoerceInt, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Path": &bexpr.FieldConfiguration{ + StructFieldName: "Path", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "LocalPathPort": &bexpr.FieldConfiguration{ + StructFieldName: "LocalPathPort", + CoerceFn: bexpr.CoerceInt, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Protocol": &bexpr.FieldConfiguration{ + StructFieldName: "Protocol", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "TLSSkipVerify": &bexpr.FieldConfiguration{ + StructFieldName: "TLSSkipVerify", + CoerceFn: bexpr.CoerceBool, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "CAFile": &bexpr.FieldConfiguration{ + StructFieldName: "CAFile", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "CACert": &bexpr.FieldConfiguration{ + StructFieldName: "CACert", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, +} + var expectedFieldConfigUpstreams bexpr.FieldConfigurations = bexpr.FieldConfigurations{ "DestinationType": &bexpr.FieldConfiguration{ StructFieldName: "DestinationType", @@ -127,6 +178,10 @@ var expectedFieldConfigConnectProxyConfig bexpr.FieldConfigurations = bexpr.Fiel StructFieldName: "MeshGateway", SubFields: expectedFieldConfigMeshGatewayConfig, }, + "Expose": &bexpr.FieldConfiguration{ + StructFieldName: "Expose", + SubFields: expectedFieldConfigExposeConfig, + }, } var expectedFieldConfigServiceConnect bexpr.FieldConfigurations = bexpr.FieldConfigurations{ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden index a911cd1890b3..41269e441840 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden @@ -16,6 +16,7 @@ { "name": "envoy.http_connection_manager", "config": { + "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden b/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden index a911cd1890b3..41269e441840 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden @@ -16,6 +16,7 @@ { "name": "envoy.http_connection_manager", "config": { + "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden b/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden index 6994537d4f4e..0445877995cb 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden @@ -16,6 +16,7 @@ { "name": "envoy.http_connection_manager", "config": { + "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/api/agent.go b/api/agent.go index 1ef331247fd9..7a5c4272096a 100644 --- a/api/agent.go +++ b/api/agent.go @@ -103,6 +103,7 @@ type AgentServiceConnectProxyConfig struct { Config map[string]interface{} `json:",omitempty" bexpr:"-"` Upstreams []Upstream `json:",omitempty"` MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` } // AgentMember represents a cluster member known to the agent diff --git a/api/config_entry.go b/api/config_entry.go index 1588f2eed8ef..f91245454b27 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -56,6 +56,41 @@ type MeshGatewayConfig struct { Mode MeshGatewayMode `json:",omitempty"` } +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks bool `json:",omitempty"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `json:",omitempty"` +} + +type Path struct { + // ListenerPort defines the port of the proxy's listener for exposed paths. + ListenerPort int `json:",omitempty"` + + // Path is the path to expose through the proxy, ie. "/metrics." + Path string `json:",omitempty"` + + // LocalPathPort is the port that the service is listening on for the given path. + LocalPathPort int `json:",omitempty"` + + // Protocol describes the upstream's service protocol. + // Valid values are "http" and "http2", defaults to "http" + Protocol string `json:",omitempty"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify bool `json:",omitempty"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile string `json:",omitempty"` + + // CACert contains the PEM encoded CA file read from CAFile + CACert string `json:",omitempty"` +} + type ServiceConfigEntry struct { Kind string Name string From 90c874735a13b5f84c4ddb283dbe27131ecf5fa1 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 16:44:53 -0600 Subject: [PATCH 19/80] Move expose config validation to NodeService func --- agent/agent_endpoint.go | 2 +- agent/config/builder.go | 3 - agent/structs/connect_proxy_config.go | 30 +--- agent/structs/connect_proxy_config_test.go | 158 --------------------- agent/structs/structs.go | 30 ++++ agent/structs/structs_test.go | 50 +++++++ agent/structs/testing_catalog.go | 26 ++++ agent/xds/clusters.go | 1 + agent/xds/listeners.go | 1 + 9 files changed, 116 insertions(+), 185 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 78a8df6a69d2..e1581b84a2f6 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -24,7 +24,7 @@ import ( "github.com/hashicorp/consul/lib/file" "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/types" - "github.com/hashicorp/go-bexpr" + bexpr "github.com/hashicorp/go-bexpr" "github.com/hashicorp/logutils" "github.com/hashicorp/serf/coordinate" "github.com/hashicorp/serf/serf" diff --git a/agent/config/builder.go b/agent/config/builder.go index 731f3a72a335..4aaa474d0899 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1362,9 +1362,6 @@ func (b *Builder) exposeConfVal(v *ExposeConfig) structs.ExposeConfig { out.Checks = b.boolVal(v.Checks) out.Paths = b.pathsVal(v.Paths) - if err := out.Finalize(); err != nil { - b.err = multierror.Append(b.err, err) - } return out } diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index db9260070c10..05d11fca2c80 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -3,11 +3,10 @@ package structs import ( "encoding/json" "fmt" - "io/ioutil" - "strings" - "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" + "io/ioutil" + "log" ) const ( @@ -360,36 +359,21 @@ type Path struct { } // Finalize validates ExposeConfig and sets default values -func (e *ExposeConfig) Finalize() error { - var known = make(map[string]bool) +func (e *ExposeConfig) Finalize(l *log.Logger) { for i := 0; i < len(e.Paths); i++ { path := &e.Paths[i] - if seen := known[path.Path]; seen { - return fmt.Errorf("duplicate paths exposed") - } - known[path.Path] = true - - if path.ListenerPort <= 0 || path.ListenerPort > 65535 { - return fmt.Errorf("invalid listener port: %d", path.ListenerPort) + if path.Protocol == "" { + path.Protocol = defaultExposeProtocol } if path.CAFile != "" { b, err := ioutil.ReadFile(path.CAFile) if err != nil { - return fmt.Errorf("failed to read CAFile '%s': %v", path.CAFile, err) + l.Printf("[WARN] envoy: failed to read CAFile '%s': %v", path.CAFile, err) + continue } path.CACert = string(b) } - - path.Protocol = strings.ToLower(path.Protocol) - if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { - return fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", - path.Protocol, path.Path) - } - if path.Protocol == "" { - path.Protocol = defaultExposeProtocol - } } - return nil } diff --git a/agent/structs/connect_proxy_config_test.go b/agent/structs/connect_proxy_config_test.go index 6326ab253c33..bc1dd0f50d17 100644 --- a/agent/structs/connect_proxy_config_test.go +++ b/agent/structs/connect_proxy_config_test.go @@ -3,9 +3,6 @@ package structs import ( "encoding/json" "fmt" - "github.com/stretchr/testify/assert" - "io/ioutil" - "os" "testing" "github.com/hashicorp/consul/api" @@ -273,158 +270,3 @@ func TestValidateMeshGatewayMode(t *testing.T) { }) } } - -func TestExposeConfig_Finalize(t *testing.T) { - t.Parallel() - - tmpDir, err := ioutil.TempDir("", "exposeconfig_") - if err != nil { - t.Fatalf("failed to create tempdir: %v", err) - } - defer os.RemoveAll(tmpDir) - - tmpFile, err := ioutil.TempFile(tmpDir, "CAFile") - if err != nil { - t.Fatalf("failed to create tempfile: %v", err) - } - defer tmpFile.Close() - - type fields struct { - Checks bool - Paths []Path - } - tests := []struct { - name string - fields fields - want ExposeConfig - wantErr bool - errMsg string - }{ - { - name: "duplicate path", - fields: fields{ - Paths: []Path{ - { - LocalPathPort: 80, - ListenerPort: 80, - Path: "/metrics", - }, - { - LocalPathPort: 80, - ListenerPort: 80, - Path: "/metrics", - }, - }, - }, - wantErr: true, - errMsg: "duplicate paths exposed", - }, - { - name: "negative listener port", - fields: fields{ - Paths: []Path{ - { - ListenerPort: -1, - }, - }, - }, - wantErr: true, - errMsg: "invalid listener port: -1", - }, - { - name: "listener port too large", - fields: fields{ - Paths: []Path{ - { - ListenerPort: 65536, - }, - }, - }, - wantErr: true, - errMsg: "invalid listener port: 65536", - }, - { - name: "protocol not supported", - fields: fields{ - Paths: []Path{ - { - Path: "/metrics", - LocalPathPort: 80, - ListenerPort: 80, - Protocol: "tcp", - }, - }, - }, - wantErr: true, - errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", - }, - { - name: "protocol not supported", - fields: fields{ - Paths: []Path{ - { - Path: "/metrics", - LocalPathPort: 80, - ListenerPort: 80, - Protocol: "tcp", - }, - }, - }, - wantErr: true, - errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", - }, - { - name: "default to http when no protocol", - fields: fields{ - Paths: []Path{ - { - LocalPathPort: 80, - ListenerPort: 80, - }, - }, - }, - wantErr: false, - want: ExposeConfig{ - Paths: []Path{ - {LocalPathPort: 80, ListenerPort: 80, Protocol: "http"}, - }, - }, - }, - { - name: "lowercase protocol", - fields: fields{ - Paths: []Path{ - { - LocalPathPort: 80, - ListenerPort: 80, - Protocol: "HTTP2", - }, - }, - }, - wantErr: false, - want: ExposeConfig{ - Paths: []Path{ - {LocalPathPort: 80, ListenerPort: 80, Protocol: "http2"}, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - e := &ExposeConfig{ - Checks: tt.fields.Checks, - Paths: tt.fields.Paths, - } - err := e.Finalize() - if (err != nil) != tt.wantErr { - t.Errorf("Finalize() got error = %v, wantErr %v", err, tt.wantErr) - } - if tt.wantErr && tt.errMsg != err.Error() { - t.Errorf("Finalize() got error: '%v', want: '%s'", err, tt.errMsg) - } - if !tt.wantErr { - assert.Equal(t, &tt.want, e) - } - }) - } -} diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 7880fef0b486..145b53fe69bc 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "net" + "os" "reflect" "regexp" "sort" @@ -946,6 +947,35 @@ func (s *NodeService) Validate() error { } bindAddrs[addr] = struct{}{} } + var known = make(map[string]bool) + for _, path := range s.Proxy.Expose.Paths { + if path.Path == "" { + result = multierror.Append(result, fmt.Errorf("empty path exposed")) + } + + if seen := known[path.Path]; seen { + result = multierror.Append(result, fmt.Errorf("duplicate paths exposed")) + } + known[path.Path] = true + + if path.ListenerPort <= 0 || path.ListenerPort > 65535 { + result = multierror.Append(result, fmt.Errorf("invalid listener port: %d", path.ListenerPort)) + } + + if path.CAFile != "" { + _, err := os.Stat(path.CAFile) + if err != nil { + result = multierror.Append(result, fmt.Errorf("failed to find CAFile '%s': %v", path.CAFile, err)) + } + } + + path.Protocol = strings.ToLower(path.Protocol) + if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { + result = multierror.Append(result, + fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", + path.Protocol, path.Path)) + } + } } // MeshGateway validation diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index 1549ef548bbe..333cdcd7ff19 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -414,6 +414,56 @@ func TestStructs_NodeService_ValidateMeshGateway(t *testing.T) { } } +func TestStructs_NodeService_ValidateExposeConfig(t *testing.T) { + type testCase struct { + Modify func(*NodeService) + Err string + } + cases := map[string]testCase{ + "valid": { + func(x *NodeService) {}, + "", + }, + "empty path": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].Path = "" }, + "empty path exposed", + }, + "invalid port negative": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].ListenerPort = -1 }, + "invalid listener port", + }, + "invalid port too large": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].ListenerPort = 65536 }, + "invalid listener port", + }, + "duplicate paths": { + func(x *NodeService) { + x.Proxy.Expose.Paths[0].Path = "/metrics" + x.Proxy.Expose.Paths[1].Path = "/metrics" + }, + "duplicate paths exposed", + }, + "protocol not supported": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].Protocol = "foo" }, + "protocol 'foo' not supported for path", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ns := TestNodeServiceExpose(t) + tc.Modify(ns) + + err := ns.Validate() + if tc.Err == "" { + require.NoError(t, err) + } else { + require.Contains(t, strings.ToLower(err.Error()), strings.ToLower(tc.Err)) + } + }) + } +} + func TestStructs_NodeService_ValidateConnectProxy(t *testing.T) { cases := []struct { Name string diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index e1f847bf6f5a..26ba4a6c6100 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -50,6 +50,32 @@ func TestNodeServiceProxy(t testing.T) *NodeService { } } +func TestNodeServiceExpose(t testing.T) *NodeService { + return &NodeService{ + Kind: ServiceKindConnectProxy, + Service: "test-svc", + Address: "localhost", + Port: 8080, + Proxy: ConnectProxyConfig{ + DestinationServiceName: "web", + Expose: ExposeConfig{ + Paths: []Path{ + { + Path: "/foo", + LocalPathPort: 80, + ListenerPort: 80, + }, + { + Path: "/bar", + LocalPathPort: 80, + ListenerPort: 80, + }, + }, + }, + }, + } +} + // TestNodeServiceMeshGateway returns a *NodeService representing a valid Mesh Gateway func TestNodeServiceMeshGateway(t testing.T) *NodeService { return TestNodeServiceMeshGatewayWithAddrs(t, diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 70b2fa87a5fb..bb2e93aa5a47 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -74,6 +74,7 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } } + cfgSnap.Proxy.Expose.Finalize(s.Logger) paths := cfgSnap.Proxy.Expose.Paths // Add service health checks to the list of paths to create clusters for if needed diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 28269b5216c0..62a54b78039f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -74,6 +74,7 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps resources[i+1] = upstreamListener } + cfgSnap.Proxy.Expose.Finalize(s.Logger) paths := cfgSnap.Proxy.Expose.Paths // Add service health checks to the list of paths to create listeners for if needed From 9966c7004c76a73bc05e0da5a42b9f24829ccb65 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 16:50:21 -0600 Subject: [PATCH 20/80] Avoid panic in expose.checks check --- agent/agent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 2171feda99e1..90bbd426f543 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2432,7 +2432,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, TLSClientConfig: tlsClientConfig, } - if service.Proxy.Expose.Checks { + if service != nil && service.Proxy.Expose.Checks { port, err := a.listenerPort(service.ID, string(http.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) @@ -2495,7 +2495,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, TLSClientConfig: tlsClientConfig, } - if service.Proxy.Expose.Checks { + if service != nil && service.Proxy.Expose.Checks { port, err := a.listenerPort(service.ID, string(grpc.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) From 127d944d21f5c27a0c9adb5a084d43f3de69d390 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 21 Aug 2019 23:48:35 -0600 Subject: [PATCH 21/80] Add service.proxy.config types --- agent/config/config.go | 1 + agent/xds/config.go | 53 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/agent/config/config.go b/agent/config/config.go index 50deea2cb878..8fbc20db6429 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -87,6 +87,7 @@ func Parse(data string, format string) (c Config, err error) { "watches", "service.connect.proxy.config.upstreams", // Deprecated "services.connect.proxy.config.upstreams", // Deprecated + "services.connect.proxy.config.expose.paths", "service.connect.proxy.upstreams", "services.connect.proxy.upstreams", "service.proxy.upstreams", diff --git a/agent/xds/config.go b/agent/xds/config.go index b9894f5cb61c..d415fee0bbb0 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -139,7 +139,7 @@ func ParseUpstreamConfigNoDefaults(m map[string]interface{}) (UpstreamConfig, er return cfg, err } -// ParseUpstreamConfig returns the UpstreamConfig parsed from the an opaque map. +// ParseUpstreamConfig returns the UpstreamConfig parsed from an opaque map. // If an error occurs during parsing it is returned along with the default // config this allows caller to choose whether and how to report the error. func ParseUpstreamConfig(m map[string]interface{}) (UpstreamConfig, error) { @@ -155,3 +155,54 @@ func ParseUpstreamConfig(m map[string]interface{}) (UpstreamConfig, error) { } return cfg, err } + +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks bool `mapstructure:"checks"` + + // Port defines the port of the proxy's listener for exposed paths. + Port int `mapstructure:"port"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `mapstructure:"paths"` +} + +type Path struct { + // Path is the path to expose through the proxy, ie. "/metrics." + Path string `mapstructure:"path"` + + // Port is the port that the service is listening on for the given path. + Port int `mapstructure:"port"` + + // Protocol describes the upstream's service protocol. + // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". + Protocol string `mapstructure:"protocol"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify bool `mapstructure:"tls_skip_verify"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile string `mapstructure:"ca_file"` +} + +// ParseExposeConfig returns the ExposeConfig parsed from an opaque map. +// If an error occurs during parsing it is returned along with the default +// config this allows caller to choose whether and how to report the error. +func ParseExposeConfig(m map[string]interface{}) (ExposeConfig, error) { + var cfg ExposeConfig + err := mapstructure.WeakDecode(m, &cfg) + if cfg.Port == 0 { + cfg.Port = 21500 + } + for _, path := range cfg.Paths { + if path.Protocol == "" { + path.Protocol = "http1.1" + } else { + path.Protocol = strings.ToLower(path.Protocol) + } + } + return cfg, err +} From 7fd306d1bd5c23ffdcb6260ae31b7fb417171629 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 27 Aug 2019 23:43:37 -0600 Subject: [PATCH 22/80] Add cache-type for service HTTP checks --- agent/agent.go | 81 ++++++++++++-- agent/agent_endpoint.go | 2 +- agent/cache-types/service_checks.go | 158 ++++++++++++++++++++++++++++ agent/checks/check.go | 48 +++++++-- agent/local/state.go | 4 +- agent/service_checks_test.go | 152 ++++++++++++++++++++++++++ agent/structs/structs.go | 8 ++ 7 files changed, 429 insertions(+), 24 deletions(-) create mode 100644 agent/cache-types/service_checks.go create mode 100644 agent/service_checks_test.go diff --git a/agent/agent.go b/agent/agent.go index 74c63c89334b..255615c7b460 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -20,7 +20,7 @@ import ( "google.golang.org/grpc" - metrics "github.com/armon/go-metrics" + "github.com/armon/go-metrics" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/ae" "github.com/hashicorp/consul/agent/cache" @@ -42,8 +42,8 @@ import ( "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/consul/types" - multierror "github.com/hashicorp/go-multierror" - uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-uuid" "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/hashicorp/serf/serf" @@ -1806,7 +1806,7 @@ func (a *Agent) ResumeSync() { // syncPausedCh returns either a channel or nil. If nil sync is not paused. If // non-nil, the channel will be closed when sync resumes. -func (a *Agent) syncPausedCh() <-chan struct{} { +func (a *Agent) SyncPausedCh() <-chan struct{} { a.syncMu.Lock() defer a.syncMu.Unlock() return a.syncCh @@ -2076,7 +2076,7 @@ func (a *Agent) addServiceInternal(service *structs.NodeService, chkTypes []*str } // cleanup, store the ids of services and checks that weren't previously - // registered so we clean them up if somthing fails halfway through the + // registered so we clean them up if something fails halfway through the // process. var cleanupServices []string var cleanupChecks []types.CheckID @@ -2367,6 +2367,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, ttl := &checks.CheckTTL{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, TTL: chkType.TTL, Logger: a.logger, OutputMaxSize: maxOutputSize, @@ -2397,6 +2398,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, http := &checks.CheckHTTP{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, HTTP: chkType.HTTP, Header: chkType.Header, Method: chkType.Method, @@ -2421,12 +2423,13 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } tcp := &checks.CheckTCP{ - Notify: a.State, - CheckID: check.CheckID, - TCP: chkType.TCP, - Interval: chkType.Interval, - Timeout: chkType.Timeout, - Logger: a.logger, + Notify: a.State, + CheckID: check.CheckID, + ServiceID: check.ServiceID, + TCP: chkType.TCP, + Interval: chkType.Interval, + Timeout: chkType.Timeout, + Logger: a.logger, } tcp.Start() a.checkTCPs[check.CheckID] = tcp @@ -2450,6 +2453,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, grpc := &checks.CheckGRPC{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, GRPC: chkType.GRPC, Interval: chkType.Interval, Timeout: chkType.Timeout, @@ -2483,6 +2487,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, dockerCheck := &checks.CheckDocker{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, DockerContainerID: chkType.DockerContainerID, Shell: chkType.Shell, ScriptArgs: chkType.ScriptArgs, @@ -2509,6 +2514,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, monitor := &checks.CheckMonitor{ Notify: a.State, CheckID: check.CheckID, + ServiceID: check.ServiceID, ScriptArgs: chkType.ScriptArgs, Interval: chkType.Interval, Timeout: chkType.Timeout, @@ -2551,6 +2557,16 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, return fmt.Errorf("Check type is not valid") } + // Notify channel that watches for service state changes + // This is a non-blocking send to avoid synchronizing on a large number of check updates + s := a.State.ServiceState(check.ServiceID) + if s != nil && !s.Deleted { + select { + case s.WatchCh <- struct{}{}: + default: + } + } + if chkType.DeregisterCriticalServiceAfter > 0 { timeout := chkType.DeregisterCriticalServiceAfter if timeout < a.config.CheckDeregisterIntervalMin { @@ -2583,6 +2599,22 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return fmt.Errorf("CheckID missing") } + // Notify channel that watches for service state changes + // This is a non-blocking send to avoid synchronizing on a large number of check updates + var svcID string + for _, c := range a.State.Checks() { + if c.CheckID == checkID { + svcID = c.ServiceID + } + } + s := a.State.ServiceState(svcID) + if s != nil && !s.Deleted { + select { + case s.WatchCh <- struct{}{}: + default: + } + } + a.cancelCheckMonitors(checkID) a.State.RemoveCheck(checkID) @@ -2598,6 +2630,21 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return nil } +func (a *Agent) ServiceHTTPChecks(serviceID string) []structs.CheckType { + var chkTypes = make([]structs.CheckType, 0) + for _, c := range a.checkHTTPs { + if c.ServiceID == serviceID { + chkTypes = append(chkTypes, c.CheckType()) + } + } + for _, c := range a.checkGRPCs { + if c.ServiceID == serviceID { + chkTypes = append(chkTypes, c.CheckType()) + } + } + return chkTypes +} + // resolveProxyCheckAddress returns the best address to use for a TCP check of // the proxy's public listener. It expects the input to already have default // values populated by applyProxyConfigDefaults. It may return an empty string @@ -3465,4 +3512,16 @@ func (a *Agent) registerCache() { RefreshTimer: 0 * time.Second, RefreshTimeout: 10 * time.Minute, }) + + a.cache.RegisterType(cachetype.ServiceHTTPChecksName, &cachetype.ServiceHTTPChecks{ + Agent: a, + }, &cache.RegisterOptions{ + Refresh: true, + RefreshTimer: 0 * time.Second, + RefreshTimeout: 10 * time.Minute, + }) +} + +func (a *Agent) LocalState() *local.State { + return a.State } diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 6653704025ca..c7a887e465b7 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -1344,7 +1344,7 @@ func (s *HTTPServer) agentLocalBlockingQuery(resp http.ResponseWriter, hash stri // case it's likely that local state just got unloaded and may or may not be // reloaded yet. Wait a short amount of time for Sync to resume to ride out // typical config reloads. - if syncPauseCh := s.agent.syncPausedCh(); syncPauseCh != nil { + if syncPauseCh := s.agent.SyncPausedCh(); syncPauseCh != nil { select { case <-syncPauseCh: case <-timeout.C: diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go new file mode 100644 index 000000000000..bd20c3959200 --- /dev/null +++ b/agent/cache-types/service_checks.go @@ -0,0 +1,158 @@ +package cachetype + +import ( + "fmt" + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/agent/local" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/lib" + "github.com/hashicorp/go-memdb" + "github.com/mitchellh/hashstructure" + "strings" + "time" +) + +// Recommended name for registration. +const ServiceHTTPChecksName = "service-http-checks" + +type Agent interface { + ServiceHTTPChecks(id string) []structs.CheckType + LocalState() *local.State + SyncPausedCh() <-chan struct{} +} + +// ServiceHTTPChecks supports fetching discovering checks in the local state +type ServiceHTTPChecks struct { + Agent Agent +} + +func (c *ServiceHTTPChecks) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { + var result cache.FetchResult + + // The request should be a CatalogDatacentersRequest. + reqReal, ok := req.(*ServiceHTTPChecksRequest) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: got wrong request type: %T, want: ServiceHTTPChecksRequest", req) + } + + var lastChecks *[]structs.CheckType + var lastHash string + var err error + + // Hash last known result as a baseline + if opts.LastResult != nil { + lastChecks, ok = opts.LastResult.Value.(*[]structs.CheckType) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: got wrong request type: %T, want: ServiceHTTPChecksRequest", req) + } + lastHash, err = hashChecks(*lastChecks) + if err != nil { + return result, fmt.Errorf("Internal cache failure: %v", err) + } + } + + var wait time.Duration + + // Adjust wait based on documented limits: https://www.consul.io/api/features/blocking.html + switch wait = reqReal.MaxQueryTime; { + case wait == 0*time.Second: + wait = 5 * time.Minute + case wait > 10*time.Minute: + wait = 10 * time.Minute + } + timeout := time.NewTimer(wait + lib.RandomStagger(wait/16)) + + var resp []structs.CheckType + var hash string + +WATCH_LOOP: + for { + // Must reset this every loop in case the Watch set is already closed but + // hash remains same. In that case we'll need to re-block on ws.Watch() + ws := memdb.NewWatchSet() + + svcState := c.Agent.LocalState().ServiceState(reqReal.ServiceID) + if svcState == nil { + return result, fmt.Errorf("Internal cache failure: service '%s' not in agent state", reqReal.ServiceID) + } + + // WatchCh will receive updates on service (de)registrations and check (de)registrations + ws.Add(svcState.WatchCh) + + resp = c.Agent.ServiceHTTPChecks(reqReal.ServiceID) + + hash, err := hashChecks(resp) + if err != nil { + return result, fmt.Errorf("Internal cache failure: %v", err) + } + + // Return immediately if the hash is different or the Watch returns true (indicating timeout fired). + if lastHash != hash || ws.Watch(timeout.C) { + break + } + + // Watch returned false indicating a change was detected, loop and repeat + // the call to ServiceHTTPChecks to load the new value. + // If agent sync is paused it means local state is being bulk-edited e.g. config reload. + if syncPauseCh := c.Agent.SyncPausedCh(); syncPauseCh != nil { + // Wait for pause to end or for the timeout to elapse. + select { + case <-syncPauseCh: + case <-timeout.C: + break WATCH_LOOP + } + } + } + + result.Value = &resp + + // Below is a purely synthetic index to keep the caching happy. + if opts.LastResult == nil { + result.Index = 1 + return result, nil + } + + result.Index = opts.LastResult.Index + if lastHash == "" || hash != lastHash { + result.Index += 1 + } + return result, nil +} + +func (c *ServiceHTTPChecks) SupportsBlocking() bool { + return true +} + +// ServiceHTTPChecksRequest is the cache.Request implementation for the +// ServiceHTTPChecks cache type. This is implemented here and not in structs +// since this is only used for cache-related requests and not forwarded +// directly to any Consul servers. +type ServiceHTTPChecksRequest struct { + ServiceID string + MinQueryIndex uint64 + MaxQueryTime time.Duration +} + +func (s *ServiceHTTPChecksRequest) CacheInfo() cache.RequestInfo { + return cache.RequestInfo{ + Token: "", + Key: ServiceHTTPChecksName + ":" + s.ServiceID, + Datacenter: "", + MinIndex: s.MinQueryIndex, + Timeout: s.MaxQueryTime, + } +} + +func hashChecks(checks []structs.CheckType) (string, error) { + var b strings.Builder + for _, check := range checks { + raw, err := hashstructure.Hash(check, nil) + if err != nil { + return "", fmt.Errorf("failed to hash check '%s': %v", check.CheckID, err) + } + fmt.Fprintf(&b, "%x", raw) + } + return b.String(), nil +} diff --git a/agent/checks/check.go b/agent/checks/check.go index a0816eb1246e..8a8a69352dff 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -3,6 +3,7 @@ package checks import ( "crypto/tls" "fmt" + "github.com/hashicorp/consul/agent/structs" "io" "io/ioutil" "log" @@ -58,6 +59,7 @@ type CheckNotifier interface { type CheckMonitor struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string Script string ScriptArgs []string Interval time.Duration @@ -210,10 +212,11 @@ func (c *CheckMonitor) check() { // but upon the TTL expiring, the check status is // automatically set to critical. type CheckTTL struct { - Notify CheckNotifier - CheckID types.CheckID - TTL time.Duration - Logger *log.Logger + Notify CheckNotifier + CheckID types.CheckID + ServiceID string + TTL time.Duration + Logger *log.Logger timer *time.Timer @@ -308,6 +311,7 @@ func (c *CheckTTL) SetStatus(status, output string) string { type CheckHTTP struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string HTTP string Header map[string][]string Method string @@ -323,6 +327,18 @@ type CheckHTTP struct { stopLock sync.Mutex } +func (c *CheckHTTP) CheckType() structs.CheckType { + return structs.CheckType{ + CheckID: c.CheckID, + HTTP: c.HTTP, + Method: c.Method, + Header: c.Header, + Interval: c.Interval, + Timeout: c.Timeout, + OutputMaxSize: c.OutputMaxSize, + } +} + // Start is used to start an HTTP check. // The check runs until stop is called func (c *CheckHTTP) Start() { @@ -456,12 +472,13 @@ func (c *CheckHTTP) check() { // The check is passing if the connection succeeds // The check is critical if the connection returns an error type CheckTCP struct { - Notify CheckNotifier - CheckID types.CheckID - TCP string - Interval time.Duration - Timeout time.Duration - Logger *log.Logger + Notify CheckNotifier + CheckID types.CheckID + ServiceID string + TCP string + Interval time.Duration + Timeout time.Duration + Logger *log.Logger dialer *net.Dialer stop bool @@ -537,6 +554,7 @@ func (c *CheckTCP) check() { type CheckDocker struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string Script string ScriptArgs []string DockerContainerID string @@ -656,6 +674,7 @@ func (c *CheckDocker) doCheck() (string, *circbuf.Buffer, error) { type CheckGRPC struct { Notify CheckNotifier CheckID types.CheckID + ServiceID string GRPC string Interval time.Duration Timeout time.Duration @@ -668,6 +687,15 @@ type CheckGRPC struct { stopLock sync.Mutex } +func (c *CheckGRPC) CheckType() structs.CheckType { + return structs.CheckType{ + CheckID: c.CheckID, + GRPC: c.GRPC, + Interval: c.Interval, + Timeout: c.Timeout, + } +} + func (c *CheckGRPC) Start() { c.stopLock.Lock() defer c.stopLock.Unlock() diff --git a/agent/local/state.go b/agent/local/state.go index 7b489782793b..dda3843f009a 100644 --- a/agent/local/state.go +++ b/agent/local/state.go @@ -50,7 +50,7 @@ type ServiceState struct { // but has not been removed on the server yet. Deleted bool - // WatchCh is closed when the service state changes suitable for use in a + // WatchCh is closed when the service state changes. Suitable for use in a // memdb.WatchSet when watching agent local changes with hash-based blocking. WatchCh chan struct{} } @@ -366,7 +366,7 @@ func (l *State) SetServiceState(s *ServiceState) { } func (l *State) setServiceStateLocked(s *ServiceState) { - s.WatchCh = make(chan struct{}) + s.WatchCh = make(chan struct{}, 1) old, hasOld := l.services[s.Service.ID] l.services[s.Service.ID] = s diff --git a/agent/service_checks_test.go b/agent/service_checks_test.go new file mode 100644 index 000000000000..083fc6570c3c --- /dev/null +++ b/agent/service_checks_test.go @@ -0,0 +1,152 @@ +package agent + +import ( + "context" + "github.com/hashicorp/consul/agent/cache" + cachetype "github.com/hashicorp/consul/agent/cache-types" + "github.com/hashicorp/consul/agent/checks" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/testrpc" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + + service := structs.NodeService{ + ID: "web", + Service: "web", + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ch := make(chan cache.UpdateEvent) + + // Watch for service check updates + err := a.cache.Notify(ctx, cachetype.ServiceHTTPChecksName, &cachetype.ServiceHTTPChecksRequest{ + ServiceID: service.ID, + }, "service-checks:"+service.ID, ch) + if err != nil { + t.Fatalf("failed to set cache notification: %v", err) + } + + chkTypes := []*structs.CheckType{ + { + CheckID: "http-check", + HTTP: "localhost:8080/health", + Interval: 5 * time.Second, + OutputMaxSize: checks.DefaultBufSize, + }, + { + CheckID: "grpc-check", + GRPC: "localhost:9090/v1.Health", + Interval: 5 * time.Second, + }, + { + CheckID: "ttl-check", + TTL: 10 * time.Second, + }, + } + + // Adding first TTL type should lead to a timeout, since only HTTP-based checks are watched + if err := a.AddService(&service, chkTypes[2:], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add service: %v", err) + } + + var val cache.UpdateEvent + select { + case val = <-ch: + t.Fatal("unexpected cache update, wanted HTTP checks, got TTL") + case <-time.After(100 * time.Millisecond): + } + + // Adding service with HTTP check should lead notification for check + if err := a.AddService(&service, chkTypes[0:1], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add service: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok := val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + want := chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Adding GRPC check should lead to a notification from the cache with both checks + hc := structs.HealthCheck{ + CheckID: chkTypes[1].CheckID, + ServiceID: service.ID, + } + if err := a.AddCheck(&hc, chkTypes[1], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add service: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok = val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + want = chkTypes[0:2] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Removing the GRPC check should leave only the HTTP check + if err := a.RemoveCheck(chkTypes[1].CheckID, false); err != nil { + t.Fatalf("failed to remove check: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok = val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + want = chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Removing the HTTP check should leave an empty list + if err := a.RemoveCheck(chkTypes[0].CheckID, false); err != nil { + t.Fatalf("failed to remove check: %v", err) + } + + select { + case val = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatal("didn't get cache update event") + } + + got, ok = val.Result.(*[]structs.CheckType) + if !ok { + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + } + if len(*got) != 0 { + t.Fatalf("expected empty result, got: %+v", got) + } +} diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 166c41bbb886..1676b85e7085 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -1122,6 +1122,14 @@ type HealthCheckDefinition struct { OutputMaxSize uint `json:",omitempty"` Timeout time.Duration `json:",omitempty"` DeregisterCriticalServiceAfter time.Duration `json:",omitempty"` + ScriptArgs []string `json:",omitempty"` + DockerContainerID string `json:",omitempty"` + Shell string `json:",omitempty"` + GRPC string `json:",omitempty"` + GRPCUseTLS bool `json:",omitempty"` + AliasNode string `json:",omitempty"` + AliasService string `json:",omitempty"` + TTL time.Duration `json:",omitempty"` } func (d *HealthCheckDefinition) MarshalJSON() ([]byte, error) { From 69c59c55e604ccb1a89bf50457eae597f6addb78 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 28 Aug 2019 23:01:44 -0600 Subject: [PATCH 23/80] Update tests for service-checks cachetype and fix hash bug --- agent/cache-types/service_checks.go | 4 +- agent/cache-types/service_checks_test.go | 178 +++++++++++++++++++++++ agent/service_checks_test.go | 57 +------- 3 files changed, 188 insertions(+), 51 deletions(-) create mode 100644 agent/cache-types/service_checks_test.go diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go index bd20c3959200..2ddae4ad68a7 100644 --- a/agent/cache-types/service_checks.go +++ b/agent/cache-types/service_checks.go @@ -55,7 +55,7 @@ func (c *ServiceHTTPChecks) Fetch(opts cache.FetchOptions, req cache.Request) (c var wait time.Duration - // Adjust wait based on documented limits: https://www.consul.io/api/features/blocking.html + // Adjust wait based on documented limits and add some jitter: https://www.consul.io/api/features/blocking.html switch wait = reqReal.MaxQueryTime; { case wait == 0*time.Second: wait = 5 * time.Minute @@ -83,7 +83,7 @@ WATCH_LOOP: resp = c.Agent.ServiceHTTPChecks(reqReal.ServiceID) - hash, err := hashChecks(resp) + hash, err = hashChecks(resp) if err != nil { return result, fmt.Errorf("Internal cache failure: %v", err) } diff --git a/agent/cache-types/service_checks_test.go b/agent/cache-types/service_checks_test.go new file mode 100644 index 000000000000..133a80cb9a45 --- /dev/null +++ b/agent/cache-types/service_checks_test.go @@ -0,0 +1,178 @@ +package cachetype_test + +import ( + "github.com/hashicorp/consul/agent/cache" + cachetype "github.com/hashicorp/consul/agent/cache-types" + "github.com/hashicorp/consul/agent/checks" + "github.com/hashicorp/consul/agent/local" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/token" + "github.com/hashicorp/consul/types" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestServiceHTTPChecks_Fetch(t *testing.T) { + chkTypes := []*structs.CheckType{ + { + CheckID: "http-check", + HTTP: "localhost:8080/health", + Interval: 5 * time.Second, + OutputMaxSize: checks.DefaultBufSize, + }, + { + CheckID: "grpc-check", + GRPC: "localhost:9090/v1.Health", + Interval: 5 * time.Second, + }, + } + + svcState := local.ServiceState{ + Service: &structs.NodeService{ + ID: "web", + }, + } + + // Create mockAgent and cache type + a := newMockAgent() + a.LocalState().SetServiceState(&svcState) + typ := cachetype.ServiceHTTPChecks{Agent: a} + + // Adding HTTP check should yield check in Fetch + if err := a.AddCheck(*chkTypes[0]); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result, err := typ.Fetch( + cache.FetchOptions{}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result.Index != 1 { + t.Fatalf("expected index of 1 after first request, got %d", result.Index) + } + + got, ok := result.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want := chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Adding GRPC check should yield both checks in Fetch + if err := a.AddCheck(*chkTypes[1]); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result2, err := typ.Fetch( + cache.FetchOptions{LastResult: &result}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result2.Index != 2 { + t.Fatalf("expected index of 2 after second request, got %d", result2.Index) + } + + got, ok = result2.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want = chkTypes[0:2] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Removing GRPC check should yield HTTP check in Fetch + if err := a.RemoveCheck(chkTypes[1].CheckID); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result3, err := typ.Fetch( + cache.FetchOptions{LastResult: &result2}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result3.Index != 3 { + t.Fatalf("expected index of 3 after third request, got %d", result3.Index) + } + + got, ok = result3.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want = chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } + + // Fetching again should yield no change in result nor index + result4, err := typ.Fetch( + cache.FetchOptions{LastResult: &result3}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + if result4.Index != 3 { + t.Fatalf("expected index of 3 after fetch timeout, got %d", result4.Index) + } + + got, ok = result4.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + want = chkTypes[0:1] + for i, c := range *got { + require.Equal(t, c, *want[i]) + } +} + +type mockAgent struct { + state *local.State + pauseCh <-chan struct{} + checks []structs.CheckType +} + +func newMockAgent() *mockAgent { + m := mockAgent{ + state: local.NewState(local.Config{NodeID: "host"}, nil, new(token.Store)), + pauseCh: make(chan struct{}), + checks: make([]structs.CheckType, 0), + } + m.state.TriggerSyncChanges = func() {} + return &m +} + +func (m *mockAgent) ServiceHTTPChecks(id string) []structs.CheckType { + return m.checks +} + +func (m *mockAgent) LocalState() *local.State { + return m.state +} + +func (m *mockAgent) SyncPausedCh() <-chan struct{} { + return m.pauseCh +} + +func (m *mockAgent) AddCheck(check structs.CheckType) error { + m.checks = append(m.checks, check) + return nil +} + +func (m *mockAgent) RemoveCheck(id types.CheckID) error { + new := make([]structs.CheckType, 0) + for _, c := range m.checks { + if c.CheckID != id { + new = append(new, c) + } + } + m.checks = new + return nil +} diff --git a/agent/service_checks_test.go b/agent/service_checks_test.go index 083fc6570c3c..b4f45ab3b5ec 100644 --- a/agent/service_checks_test.go +++ b/agent/service_checks_test.go @@ -12,6 +12,8 @@ import ( "time" ) +// Integration test for ServiceHTTPChecks cache-type +// Placed in agent pkg rather than cache-types to avoid circular dependency when importing agent.TestAgent func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { t.Parallel() @@ -63,12 +65,12 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { var val cache.UpdateEvent select { case val = <-ch: - t.Fatal("unexpected cache update, wanted HTTP checks, got TTL") + t.Fatal("got cache update for TTL check, expected timeout") case <-time.After(100 * time.Millisecond): } - // Adding service with HTTP check should lead notification for check - if err := a.AddService(&service, chkTypes[0:1], false, "", ConfigSourceLocal); err != nil { + // Adding service with HTTP checks should lead notification for them + if err := a.AddService(&service, chkTypes[0:2], false, "", ConfigSourceLocal); err != nil { t.Fatalf("failed to add service: %v", err) } @@ -80,33 +82,9 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { got, ok := val.Result.(*[]structs.CheckType) if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + t.Fatalf("notified of result of wrong type, got %T, want *[]structs.CheckType", got) } - want := chkTypes[0:1] - for i, c := range *got { - require.Equal(t, c, *want[i]) - } - - // Adding GRPC check should lead to a notification from the cache with both checks - hc := structs.HealthCheck{ - CheckID: chkTypes[1].CheckID, - ServiceID: service.ID, - } - if err := a.AddCheck(&hc, chkTypes[1], false, "", ConfigSourceLocal); err != nil { - t.Fatalf("failed to add service: %v", err) - } - - select { - case val = <-ch: - case <-time.After(100 * time.Millisecond): - t.Fatal("didn't get cache update event") - } - - got, ok = val.Result.(*[]structs.CheckType) - if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) - } - want = chkTypes[0:2] + want := chkTypes[0:2] for i, c := range *got { require.Equal(t, c, *want[i]) } @@ -124,29 +102,10 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { got, ok = val.Result.(*[]structs.CheckType) if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) + t.Fatalf("notified of result of wrong type, got %T, want *[]structs.CheckType", got) } want = chkTypes[0:1] for i, c := range *got { require.Equal(t, c, *want[i]) } - - // Removing the HTTP check should leave an empty list - if err := a.RemoveCheck(chkTypes[0].CheckID, false); err != nil { - t.Fatalf("failed to remove check: %v", err) - } - - select { - case val = <-ch: - case <-time.After(100 * time.Millisecond): - t.Fatal("didn't get cache update event") - } - - got, ok = val.Result.(*[]structs.CheckType) - if !ok { - t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) - } - if len(*got) != 0 { - t.Fatalf("expected empty result, got: %+v", got) - } } From 3e88f6b8f5f05177663bde3e9f163adf8de1a379 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 28 Aug 2019 23:10:09 -0600 Subject: [PATCH 24/80] Set up service-check notification and handling --- agent/proxycfg/snapshot.go | 1 + agent/proxycfg/state.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 37b8bf574958..47ee197858ed 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -14,6 +14,7 @@ type configSnapshotConnectProxy struct { WatchedUpstreamEndpoints map[string]map[string]structs.CheckServiceNodes WatchedGateways map[string]map[string]context.CancelFunc WatchedGatewayEndpoints map[string]map[string]structs.CheckServiceNodes + WatchedServiceChecks map[string][]structs.CheckType UpstreamEndpoints map[string]structs.CheckServiceNodes // DEPRECATED:see:WatchedUpstreamEndpoints } diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index eea31d100ff4..608c4d20915d 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -29,6 +29,7 @@ const ( serviceListWatchID = "service-list" datacentersWatchID = "datacenters" serviceResolversWatchID = "service-resolvers" + svcChecksWatchIDPrefix = cachetype.ServiceHTTPChecksName + ":" serviceIDPrefix = string(structs.UpstreamDestTypeService) + ":" preparedQueryIDPrefix = string(structs.UpstreamDestTypePreparedQuery) + ":" defaultPreparedQueryPollInterval = 30 * time.Second @@ -215,6 +216,14 @@ func (s *state) initWatchesConnectProxy() error { return err } + // Watch for service check updates + err = s.cache.Notify(s.ctx, cachetype.ServiceHTTPChecksName, &cachetype.ServiceHTTPChecksRequest{ + ServiceID: s.proxyCfg.DestinationServiceID, + }, svcChecksWatchIDPrefix+s.proxyCfg.DestinationServiceID, s.ch) + if err != nil { + return err + } + // TODO(namespaces): pull this from something like s.source.Namespace? currentNamespace := "default" @@ -353,6 +362,7 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot { snap.ConnectProxy.WatchedUpstreamEndpoints = make(map[string]map[string]structs.CheckServiceNodes) snap.ConnectProxy.WatchedGateways = make(map[string]map[string]context.CancelFunc) snap.ConnectProxy.WatchedGatewayEndpoints = make(map[string]map[string]structs.CheckServiceNodes) + snap.ConnectProxy.WatchedServiceChecks = make(map[string][]structs.CheckType) snap.ConnectProxy.UpstreamEndpoints = make(map[string]structs.CheckServiceNodes) // TODO(rb): deprecated case structs.ServiceKindMeshGateway: @@ -541,6 +551,14 @@ func (s *state) handleUpdateConnectProxy(u cache.UpdateEvent, snap *ConfigSnapsh pq := strings.TrimPrefix(u.CorrelationID, "upstream:") snap.ConnectProxy.UpstreamEndpoints[pq] = resp.Nodes + case strings.HasPrefix(u.CorrelationID, svcChecksWatchIDPrefix): + resp, ok := u.Result.(*[]structs.CheckType) + if !ok { + return fmt.Errorf("invalid type for service checks response: %T, want: *[]structs.CheckType", u.Result) + } + svcID := strings.TrimPrefix(u.CorrelationID, svcChecksWatchIDPrefix) + snap.ConnectProxy.WatchedServiceChecks[svcID] = *resp + default: return errors.New("unknown correlation ID") } From 478a01e06496217f2f5a54f624832efe94d30418 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 29 Aug 2019 16:07:11 -0600 Subject: [PATCH 25/80] Move expose config out of opaque proxy.config --- agent/config/config.go | 1 - agent/structs/connect_proxy_config.go | 77 +++++++++++++++++++++++++++ agent/xds/config.go | 51 ------------------ 3 files changed, 77 insertions(+), 52 deletions(-) diff --git a/agent/config/config.go b/agent/config/config.go index 8fbc20db6429..50deea2cb878 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -87,7 +87,6 @@ func Parse(data string, format string) (c Config, err error) { "watches", "service.connect.proxy.config.upstreams", // Deprecated "services.connect.proxy.config.upstreams", // Deprecated - "services.connect.proxy.config.expose.paths", "service.connect.proxy.upstreams", "services.connect.proxy.upstreams", "service.proxy.upstreams", diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 78c4cb93635a..0f92f332bf85 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -3,11 +3,20 @@ package structs import ( "encoding/json" "fmt" + "io/ioutil" + "strings" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" ) +const ( + defaultExposePort = 21500 + defaultExposeProtocol = "http1.1" +) + +var allowedExposeProtocols = map[string]bool{"http1.1": true, "http2": true, "grpc": true} + type MeshGatewayMode string const ( @@ -312,3 +321,71 @@ func UpstreamFromAPI(u api.Upstream) Upstream { Config: u.Config, } } + +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks bool `mapstructure:"checks"` + + // Port defines the port of the proxy's listener for exposed paths. + Port int `mapstructure:"port"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `mapstructure:"paths"` +} + +type Path struct { + // Path is the path to expose through the proxy, ie. "/metrics." + Path string `mapstructure:"path"` + + // Port is the port that the service is listening on for the given path. + Port int `mapstructure:"port"` + + // Protocol describes the upstream's service protocol. + // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". + Protocol string `mapstructure:"protocol"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify bool `mapstructure:"tls_skip_verify"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile string `mapstructure:"ca_file"` + + // CACert contains the PEM encoded CA file read from CAFile + CACert string +} + +// Finalize validates ExposeConfig and sets default values +func (e *ExposeConfig) Finalize() error { + if e.Port < 0 || e.Port > 65535 { + return fmt.Errorf("invalid port: %d", e.Port) + } + if e.Port == 0 { + e.Port = defaultExposePort + } + + var known = make(map[string]bool) + for _, path := range e.Paths { + if seen := known[path.Path]; seen { + return fmt.Errorf("duplicate paths exposed") + } + known[path.Path] = true + + b, err := ioutil.ReadFile(path.CAFile) + if err != nil { + return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + } + path.CACert = string(b) + + path.Protocol = strings.ToLower(path.Protocol) + if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { + return fmt.Errorf("protocol '%s' not recognized for path: %s", path.Protocol, path.Path) + } + if path.Protocol == "" { + path.Protocol = defaultExposeProtocol + } + } + return nil +} diff --git a/agent/xds/config.go b/agent/xds/config.go index d415fee0bbb0..b3df755cd6cd 100644 --- a/agent/xds/config.go +++ b/agent/xds/config.go @@ -155,54 +155,3 @@ func ParseUpstreamConfig(m map[string]interface{}) (UpstreamConfig, error) { } return cfg, err } - -// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. -// Users can expose individual paths and/or all HTTP/GRPC paths for checks. -type ExposeConfig struct { - // Checks defines whether paths associated with Consul checks will be exposed. - // This flag triggers exposing all HTTP and GRPC check paths registered for the service. - Checks bool `mapstructure:"checks"` - - // Port defines the port of the proxy's listener for exposed paths. - Port int `mapstructure:"port"` - - // Paths is the list of paths exposed through the proxy. - Paths []Path `mapstructure:"paths"` -} - -type Path struct { - // Path is the path to expose through the proxy, ie. "/metrics." - Path string `mapstructure:"path"` - - // Port is the port that the service is listening on for the given path. - Port int `mapstructure:"port"` - - // Protocol describes the upstream's service protocol. - // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". - Protocol string `mapstructure:"protocol"` - - // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. - TLSSkipVerify bool `mapstructure:"tls_skip_verify"` - - // CAFile is the path to the PEM encoded CA cert used to verify client certificates. - CAFile string `mapstructure:"ca_file"` -} - -// ParseExposeConfig returns the ExposeConfig parsed from an opaque map. -// If an error occurs during parsing it is returned along with the default -// config this allows caller to choose whether and how to report the error. -func ParseExposeConfig(m map[string]interface{}) (ExposeConfig, error) { - var cfg ExposeConfig - err := mapstructure.WeakDecode(m, &cfg) - if cfg.Port == 0 { - cfg.Port = 21500 - } - for _, path := range cfg.Paths { - if path.Protocol == "" { - path.Protocol = "http1.1" - } else { - path.Protocol = strings.ToLower(path.Protocol) - } - } - return cfg, err -} From c48b0cc640b633b8b593383686b4f3dc4ce76cab Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 30 Aug 2019 13:31:18 -0600 Subject: [PATCH 26/80] Inject proxy address into HTTP checks --- agent/agent.go | 42 ++++++++++ agent/agent_test.go | 107 ++++++++++++++++++++++++++ agent/checks/check.go | 13 +++- agent/structs/connect_proxy_config.go | 3 + 4 files changed, 163 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 255615c7b460..ea760f7c7a3c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strconv" "strings" "sync" @@ -2408,6 +2409,16 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, OutputMaxSize: maxOutputSize, TLSClientConfig: tlsClientConfig, } + + if service.Proxy.Expose.Checks { + addr, err := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which it shouldn't + return fmt.Errorf("failed to inject proxy addr into HTTP target") + } + http.ProxyHTTP = addr + } + http.Start() a.checkHTTPs[check.CheckID] = http @@ -3525,3 +3536,34 @@ func (a *Agent) registerCache() { func (a *Agent) LocalState() *local.State { return a.State } + +// httpInjectAddr injects a port then an IP into a URL +func httpInjectAddr(url string, ip string, port int) (string, error) { + pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" + r, err := regexp.Compile(pattern) + if err != nil { + return "", err + } + portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) + out := r.ReplaceAllString(url, portRepl) + + // Ensure that ipv6 addr is enclosed in brackets (RFC 3986) + ip = fixIPv6(ip) + addrRepl := fmt.Sprintf("${1}%s${3}${4}${5}", ip) + out = r.ReplaceAllString(out, addrRepl) + + return out, nil +} + +func fixIPv6(address string) string { + if strings.Count(address, ":") < 2 { + return address + } + if !strings.HasSuffix(address, "]") { + address = address + "]" + } + if !strings.HasPrefix(address, "[") { + address = "[" + address + } + return address +} diff --git a/agent/agent_test.go b/agent/agent_test.go index 3b1bee23fc4c..62c6dea9f22d 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3347,3 +3347,110 @@ func TestAgent_consulConfig_RaftTrailingLogs(t *testing.T) { defer a.Shutdown() require.Equal(t, uint64(812345), a.consulConfig().RaftConfig.TrailingLogs) } + +func TestAgent_httpInjectAddr(t *testing.T) { + tt := []struct { + name string + url string + ip string + port int + want string + }{ + // TODO(freddy): IPv6 checks + { + name: "localhost health", + url: "http://localhost:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "http://192.168.0.0:9090/health", + }, + { + name: "https localhost health", + url: "https://localhost:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv4 health", + url: "https://127.0.0.1:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv4 without path", + url: "https://127.0.0.1:8080", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090", + }, + { + name: "https ipv6 health", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv6 with zone", + url: "https://[::FFFF:C0A8:1%1]:5000/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv6 literal", + url: "https://[::FFFF:192.168.0.1]:5000/health", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090/health", + }, + { + name: "https ipv6 without path", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000", + ip: "192.168.0.0", + port: 9090, + want: "https://192.168.0.0:9090", + }, + { + name: "ipv6 injected into ipv6 url", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "https://[::FFFF:C0A8:1]:9090", + }, + { + name: "ipv6 with brackets injected into ipv6 url", + url: "https://[2001:db8:1f70::999:de8:7648:6e8]:5000", + ip: "[::FFFF:C0A8:1]", + port: 9090, + want: "https://[::FFFF:C0A8:1]:9090", + }, + { + name: "short domain health", + url: "http://i.co:8080/health", + ip: "192.168.0.0", + port: 9090, + want: "http://192.168.0.0:9090/health", + }, + { + name: "nested url in query", + url: "http://my.corp.com:8080/health?from=http://google.com:8080", + ip: "192.168.0.0", + port: 9090, + want: "http://192.168.0.0:9090/health?from=http://google.com:8080", + }, + } + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + got, err := httpInjectAddr(tt.url, tt.ip, tt.port) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/agent/checks/check.go b/agent/checks/check.go index 8a8a69352dff..03b8fd379183 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -325,6 +325,10 @@ type CheckHTTP struct { stop bool stopCh chan struct{} stopLock sync.Mutex + + // Set if checks are exposed through Connect proxies + // If set, this is the target of check() + ProxyHTTP string } func (c *CheckHTTP) CheckType() structs.CheckType { @@ -406,7 +410,12 @@ func (c *CheckHTTP) check() { method = "GET" } - req, err := http.NewRequest(method, c.HTTP, nil) + target := c.HTTP + if c.ProxyHTTP != "" { + target = c.HTTP + } + + req, err := http.NewRequest(method, target, nil) if err != nil { c.Logger.Printf("[WARN] agent: Check %q HTTP request failed: %s", c.CheckID, err) c.Notify.UpdateCheck(c.CheckID, api.HealthCritical, err.Error()) @@ -446,7 +455,7 @@ func (c *CheckHTTP) check() { } // Format the response body - result := fmt.Sprintf("HTTP %s %s: %s Output: %s", method, c.HTTP, resp.Status, output.String()) + result := fmt.Sprintf("HTTP %s %s: %s Output: %s", method, target, resp.Status, output.String()) if resp.StatusCode >= 200 && resp.StatusCode <= 299 { // PASSING (2xx) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 0f92f332bf85..57ad153bd27d 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -118,6 +118,9 @@ type ConnectProxyConfig struct { // MeshGateway defines the mesh gateway configuration for this upstream MeshGateway MeshGatewayConfig `json:",omitempty"` + + // Expose defines whether checks or paths are exposed through the proxy + Expose ExposeConfig `json:",omitempty"` } func (c *ConnectProxyConfig) MarshalJSON() ([]byte, error) { From d59be99c9cebb13d30674cd79d2e420c20604d5f Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 30 Aug 2019 17:26:24 -0600 Subject: [PATCH 27/80] Inject proxy address into GRPC checks --- agent/agent.go | 28 +++++++++++- agent/agent_test.go | 100 +++++++++++++++++++++++++++++++++++++++++- agent/checks/check.go | 13 +++++- agent/checks/grpc.go | 9 ++-- 4 files changed, 142 insertions(+), 8 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index ea760f7c7a3c..a9ed5200383a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2413,7 +2413,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, if service.Proxy.Expose.Checks { addr, err := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which it shouldn't + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests return fmt.Errorf("failed to inject proxy addr into HTTP target") } http.ProxyHTTP = addr @@ -2471,6 +2471,16 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, Logger: a.logger, TLSClientConfig: tlsClientConfig, } + + if service.Proxy.Expose.Checks { + addr, err := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests + return fmt.Errorf("failed to inject proxy addr into GRPC target") + } + grpc.ProxyGRPC = addr + } + grpc.Start() a.checkGRPCs[check.CheckID] = grpc @@ -3537,6 +3547,22 @@ func (a *Agent) LocalState() *local.State { return a.State } +// grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] +func grpcInjectAddr(existing string, ip string, port int) (string, error) { + pattern := "(.*)((?::)(?:[0-9]+))(.*)$" + r, err := regexp.Compile(pattern) + if err != nil { + return "", err + } + portRepl := fmt.Sprintf("${1}:%d${3}", port) + out := r.ReplaceAllString(existing, portRepl) + + addrRepl := fmt.Sprintf("%s${2}${3}", ip) + out = r.ReplaceAllString(out, addrRepl) + + return out, nil +} + // httpInjectAddr injects a port then an IP into a URL func httpInjectAddr(url string, ip string, port int) (string, error) { pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" diff --git a/agent/agent_test.go b/agent/agent_test.go index 62c6dea9f22d..22c9c58a4d5f 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3348,6 +3348,105 @@ func TestAgent_consulConfig_RaftTrailingLogs(t *testing.T) { require.Equal(t, uint64(812345), a.consulConfig().RaftConfig.TrailingLogs) } +func TestAgent_grpcInjectAddr(t *testing.T) { + tt := []struct { + name string + grpc string + ip string + port int + want string + }{ + { + name: "localhost web svc", + grpc: "localhost:8080/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "localhost no svc", + grpc: "localhost:8080", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090", + }, + { + name: "ipv4 web svc", + grpc: "127.0.0.1:8080/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "ipv4 no svc", + grpc: "127.0.0.1:8080", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090", + }, + { + name: "ipv6 no svc", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090", + }, + { + name: "ipv6 web svc", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "zone ipv6 web svc", + grpc: "::FFFF:C0A8:1%1:5000/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "ipv6 literal web svc", + grpc: "::FFFF:192.168.0.1:5000/web", + ip: "192.168.0.0", + port: 9090, + want: "192.168.0.0:9090/web", + }, + { + name: "ipv6 injected into ipv6 url", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "::FFFF:C0A8:1:9090", + }, + { + name: "ipv6 injected into ipv6 url with svc", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000/web", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "::FFFF:C0A8:1:9090/web", + }, + { + name: "ipv6 injected into ipv6 url with special", + grpc: "2001:db8:1f70::999:de8:7648:6e8:5000/service-$name:with@special:Chars", + ip: "::FFFF:C0A8:1", + port: 9090, + want: "::FFFF:C0A8:1:9090/service-$name:with@special:Chars", + }, + } + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + got, err := grpcInjectAddr(tt.grpc, tt.ip, tt.port) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) + } + }) + } +} + func TestAgent_httpInjectAddr(t *testing.T) { tt := []struct { name string @@ -3356,7 +3455,6 @@ func TestAgent_httpInjectAddr(t *testing.T) { port int want string }{ - // TODO(freddy): IPv6 checks { name: "localhost health", url: "http://localhost:8080/health", diff --git a/agent/checks/check.go b/agent/checks/check.go index 03b8fd379183..d25566ff7ae2 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -694,6 +694,10 @@ type CheckGRPC struct { stop bool stopCh chan struct{} stopLock sync.Mutex + + // Set if checks are exposed through Connect proxies + // If set, this is the target of check() + ProxyGRPC string } func (c *CheckGRPC) CheckType() structs.CheckType { @@ -734,13 +738,18 @@ func (c *CheckGRPC) run() { } func (c *CheckGRPC) check() { - err := c.probe.Check() + target := c.GRPC + if c.ProxyGRPC != "" { + target = c.ProxyGRPC + } + + err := c.probe.Check(target) if err != nil { c.Logger.Printf("[DEBUG] agent: Check %q failed: %s", c.CheckID, err.Error()) c.Notify.UpdateCheck(c.CheckID, api.HealthCritical, err.Error()) } else { c.Logger.Printf("[DEBUG] agent: Check %q is passing", c.CheckID) - c.Notify.UpdateCheck(c.CheckID, api.HealthPassing, fmt.Sprintf("gRPC check %s: success", c.GRPC)) + c.Notify.UpdateCheck(c.CheckID, api.HealthPassing, fmt.Sprintf("gRPC check %s: success", target)) } } diff --git a/agent/checks/grpc.go b/agent/checks/grpc.go index 8577ae6e7c40..4ad7b9f34bf4 100644 --- a/agent/checks/grpc.go +++ b/agent/checks/grpc.go @@ -28,7 +28,6 @@ type GrpcHealthProbe struct { func NewGrpcHealthProbe(target string, timeout time.Duration, tlsConfig *tls.Config) *GrpcHealthProbe { serverAndService := strings.SplitN(target, "/", 2) - server := serverAndService[0] request := hv1.HealthCheckRequest{} if len(serverAndService) > 1 { request.Service = serverAndService[1] @@ -43,7 +42,6 @@ func NewGrpcHealthProbe(target string, timeout time.Duration, tlsConfig *tls.Con } return &GrpcHealthProbe{ - server: server, request: &request, timeout: timeout, dialOptions: dialOptions, @@ -52,11 +50,14 @@ func NewGrpcHealthProbe(target string, timeout time.Duration, tlsConfig *tls.Con // Check if the target of this GrpcHealthProbe is healthy // If nil is returned, target is healthy, otherwise target is not healthy -func (probe *GrpcHealthProbe) Check() error { +func (probe *GrpcHealthProbe) Check(target string) error { + serverAndService := strings.SplitN(target, "/", 2) + server := serverAndService[0] + ctx, cancel := context.WithTimeout(context.Background(), probe.timeout) defer cancel() - connection, err := grpc.DialContext(ctx, probe.server, probe.dialOptions...) + connection, err := grpc.DialContext(ctx, server, probe.dialOptions...) if err != nil { return err } From 06d120cf534537fb23d5f5e1cce532839102a5d1 Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 30 Aug 2019 18:19:28 -0600 Subject: [PATCH 28/80] Set and reset check targets on proxy (de)registration --- agent/agent.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/agent/agent.go b/agent/agent.go index a9ed5200383a..c5ca7a6fc4a0 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2113,6 +2113,20 @@ func (a *Agent) addServiceInternal(service *structs.NodeService, chkTypes []*str } } + // If a proxy service wishes to expose checks, check targets need to be rerouted to the proxy listener + // This needs to be called after chkTypes are added to the agent, to avoid overwriting + if service.Proxy.Expose.Checks { + err := a.rerouteExposedChecks( + service.Proxy.DestinationServiceID, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + if err != nil { + a.logger.Println("[WARN] failed to reroute L7 checks to exposed proxy listener") + } + } else { + // Reset check targets if proxy was re-registered but no longer wants to expose checks + // If the proxy is being registered for the first time then this is a no-op + a.resetExposedChecks(service.Proxy.DestinationServiceID) + } + // Persist the service to a file if persist && a.config.DataDir != "" { if err := a.persistService(service); err != nil { @@ -2224,6 +2238,11 @@ func (a *Agent) removeServiceLocked(serviceID string, persist bool) error { a.serviceManager.RemoveService(serviceID) } + // Reset the HTTP check targets if they were exposed through a proxy + // If this is not a proxy or checks were not exposed then this is a no-op + svc := a.State.Service(serviceID) + a.resetExposedChecks(svc.Proxy.DestinationServiceID) + checks := a.State.Checks() var checkIDs []types.CheckID for id, check := range checks { @@ -3547,6 +3566,54 @@ func (a *Agent) LocalState() *local.State { return a.State } +// rerouteExposedChecks will inject proxy address into check targets +// Future calls to check() will dial the proxy listener +// The agent stateLock MUST be held for this to be called +func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPort int) error { + for _, c := range a.checkHTTPs { + if c.ServiceID != serviceID { + continue + } + addr, err := httpInjectAddr(c.HTTP, proxyAddr, proxyPort) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests + return fmt.Errorf("failed to inject proxy addr into HTTP target") + } + c.ProxyHTTP = addr + } + for _, c := range a.checkGRPCs { + if c.ServiceID != serviceID { + continue + } + addr, err := grpcInjectAddr(c.GRPC, proxyAddr, proxyPort) + if err != nil { + // The only way to get here is if the regex pattern fails to compile, which would be caught by tests + return fmt.Errorf("failed to inject proxy addr into GRPC target") + } + c.ProxyGRPC = addr + } + return nil +} + +// resetExposedChecks will set Proxy addr in HTTP checks to empty string +// Future calls to check() will use the original target c.HTTP or c.GRPC +// The agent stateLock MUST be held for this to be called +func (a *Agent) resetExposedChecks(serviceID string) { + a.stateLock.Lock() + defer a.stateLock.Unlock() + + for _, c := range a.checkHTTPs { + if c.ServiceID == serviceID { + c.ProxyHTTP = "" + } + } + for _, c := range a.checkGRPCs { + if c.ServiceID == serviceID { + c.ProxyGRPC = "" + } + } +} + // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] func grpcInjectAddr(existing string, ip string, port int) (string, error) { pattern := "(.*)((?::)(?:[0-9]+))(.*)$" From 30544e23b257ed76a1cbc6ac0689c050d88cf791 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 2 Sep 2019 13:16:27 -0600 Subject: [PATCH 29/80] Rename http1.1 in Expose.Protocol to http --- agent/structs/connect_proxy_config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 57ad153bd27d..d43edc4e280f 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -12,10 +12,10 @@ import ( const ( defaultExposePort = 21500 - defaultExposeProtocol = "http1.1" + defaultExposeProtocol = "http" ) -var allowedExposeProtocols = map[string]bool{"http1.1": true, "http2": true, "grpc": true} +var allowedExposeProtocols = map[string]bool{"http": true, "http2": true, "grpc": true} type MeshGatewayMode string @@ -347,7 +347,7 @@ type Path struct { Port int `mapstructure:"port"` // Protocol describes the upstream's service protocol. - // Valid values are "http1.1", "http2" and "grpc". Defaults to "http1.1". + // Valid values are "http", "http2" and "grpc". Defaults to "http". Protocol string `mapstructure:"protocol"` // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. From eddb16e8c471ec44bb9819df9c37db21eb933efa Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 2 Sep 2019 13:32:38 -0600 Subject: [PATCH 30/80] Only try to read CAFile if provided --- agent/structs/connect_proxy_config.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index d43edc4e280f..a40bc78ccf02 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -376,11 +376,13 @@ func (e *ExposeConfig) Finalize() error { } known[path.Path] = true - b, err := ioutil.ReadFile(path.CAFile) - if err != nil { - return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + if path.CAFile != "" { + b, err := ioutil.ReadFile(path.CAFile) + if err != nil { + return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + } + path.CACert = string(b) } - path.CACert = string(b) path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { From a4dbffc14a4ea4faed1c39907063a990dcc3cfe6 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 2 Sep 2019 15:14:25 -0600 Subject: [PATCH 31/80] Fix deadlock in resetExposedChecks --- agent/agent.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index c5ca7a6fc4a0..608bff4d80c0 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3599,9 +3599,6 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPo // Future calls to check() will use the original target c.HTTP or c.GRPC // The agent stateLock MUST be held for this to be called func (a *Agent) resetExposedChecks(serviceID string) { - a.stateLock.Lock() - defer a.stateLock.Unlock() - for _, c := range a.checkHTTPs { if c.ServiceID == serviceID { c.ProxyHTTP = "" From 67f59ef76e3f2fcb4dc3d71be2da43cf1c24e8d8 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 3 Sep 2019 20:49:16 -0600 Subject: [PATCH 32/80] Generate listener port for exposing checks --- agent/agent.go | 96 +++++++++++++++++++++++++++++++++------------ agent/agent_test.go | 10 +---- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 608bff4d80c0..56edcbae58ce 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -184,6 +184,9 @@ type Agent struct { // checkAliases maps the check ID to an associated Alias checks checkAliases map[types.CheckID]*checks.CheckAlias + // exposedPorts tracks listener ports for checks exposed through a proxy + exposedPorts map[string]int + // stateLock protects the agent state stateLock sync.Mutex @@ -2114,10 +2117,9 @@ func (a *Agent) addServiceInternal(service *structs.NodeService, chkTypes []*str } // If a proxy service wishes to expose checks, check targets need to be rerouted to the proxy listener - // This needs to be called after chkTypes are added to the agent, to avoid overwriting + // This needs to be called after chkTypes are added to the agent, to avoid being overwritten if service.Proxy.Expose.Checks { - err := a.rerouteExposedChecks( - service.Proxy.DestinationServiceID, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + err := a.rerouteExposedChecks(service.Proxy.DestinationServiceID, service.Proxy.LocalServiceAddress) if err != nil { a.logger.Println("[WARN] failed to reroute L7 checks to exposed proxy listener") } @@ -2430,11 +2432,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } if service.Proxy.Expose.Checks { - addr, err := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + port, err := a.listenerPort(service.ID, string(http.CheckID)) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into HTTP target") + a.logger.Printf("[ERR] agent: error exposing check: %s", err) + return err } + addr := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, port) http.ProxyHTTP = addr } @@ -2492,11 +2495,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } if service.Proxy.Expose.Checks { - addr, err := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, service.Proxy.Expose.Port) + port, err := a.listenerPort(service.ID, string(grpc.CheckID)) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into GRPC target") + a.logger.Printf("[ERR] agent: error exposing check: %s", err) + return err } + addr := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, port) grpc.ProxyGRPC = addr } @@ -2655,6 +2659,11 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { } } + // Delete port from allocated port set + // If checks weren't being exposed then this is a no-op + portKey := fmt.Sprintf("%s:%s", svcID, checkID) + delete(a.exposedPorts, portKey) + a.cancelCheckMonitors(checkID) a.State.RemoveCheck(checkID) @@ -2666,6 +2675,7 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return err } } + a.logger.Printf("[DEBUG] agent: removed check %q", checkID) return nil } @@ -3569,12 +3579,16 @@ func (a *Agent) LocalState() *local.State { // rerouteExposedChecks will inject proxy address into check targets // Future calls to check() will dial the proxy listener // The agent stateLock MUST be held for this to be called -func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPort int) error { +func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string) error { for _, c := range a.checkHTTPs { if c.ServiceID != serviceID { continue } - addr, err := httpInjectAddr(c.HTTP, proxyAddr, proxyPort) + port, err := a.listenerPort(serviceID, string(c.CheckID)) + if err != nil { + return err + } + addr := httpInjectAddr(c.HTTP, proxyAddr, port) if err != nil { // The only way to get here is if the regex pattern fails to compile, which would be caught by tests return fmt.Errorf("failed to inject proxy addr into HTTP target") @@ -3585,11 +3599,11 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string, proxyPo if c.ServiceID != serviceID { continue } - addr, err := grpcInjectAddr(c.GRPC, proxyAddr, proxyPort) + port, err := a.listenerPort(serviceID, string(c.CheckID)) if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into GRPC target") + return err } + addr := grpcInjectAddr(c.GRPC, proxyAddr, port) c.ProxyGRPC = addr } return nil @@ -3609,31 +3623,61 @@ func (a *Agent) resetExposedChecks(serviceID string) { c.ProxyGRPC = "" } } + for k, _ := range a.exposedPorts { + if strings.HasPrefix(k, serviceID) { + delete(a.exposedPorts, k) + } + } +} + +// listenerPort allocates a port from the configured range +func (a *Agent) listenerPort(svcID, checkID string) (int, error) { + key := fmt.Sprintf("%s:%s", svcID, checkID) + if a.exposedPorts == nil { + a.exposedPorts = make(map[string]int) + } + if p, ok := a.exposedPorts[key]; ok { + return p, nil + } + + allocated := make(map[int]bool) + for _, v := range a.exposedPorts { + allocated[v] = true + } + + var port int + for i := a.config.ExposeMinPort; i < a.config.ExposeMaxPort; i++ { + port = a.config.ExposeMinPort + i + if !allocated[port] { + break + } + } + if port == 0 { + return 0, fmt.Errorf("no ports available to expose '%s'", checkID) + } + + return port, nil } // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] -func grpcInjectAddr(existing string, ip string, port int) (string, error) { +func grpcInjectAddr(existing string, ip string, port int) string { pattern := "(.*)((?::)(?:[0-9]+))(.*)$" - r, err := regexp.Compile(pattern) - if err != nil { - return "", err - } + r := regexp.MustCompile(pattern) + portRepl := fmt.Sprintf("${1}:%d${3}", port) out := r.ReplaceAllString(existing, portRepl) addrRepl := fmt.Sprintf("%s${2}${3}", ip) out = r.ReplaceAllString(out, addrRepl) - return out, nil + return out } // httpInjectAddr injects a port then an IP into a URL -func httpInjectAddr(url string, ip string, port int) (string, error) { +func httpInjectAddr(url string, ip string, port int) string { pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" - r, err := regexp.Compile(pattern) - if err != nil { - return "", err - } + r := regexp.MustCompile(pattern) + portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) out := r.ReplaceAllString(url, portRepl) @@ -3642,7 +3686,7 @@ func httpInjectAddr(url string, ip string, port int) (string, error) { addrRepl := fmt.Sprintf("${1}%s${3}${4}${5}", ip) out = r.ReplaceAllString(out, addrRepl) - return out, nil + return out } func fixIPv6(address string) string { diff --git a/agent/agent_test.go b/agent/agent_test.go index 22c9c58a4d5f..992341c8b2d4 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3436,10 +3436,7 @@ func TestAgent_grpcInjectAddr(t *testing.T) { } for _, tt := range tt { t.Run(tt.name, func(t *testing.T) { - got, err := grpcInjectAddr(tt.grpc, tt.ip, tt.port) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + got := grpcInjectAddr(tt.grpc, tt.ip, tt.port) if got != tt.want { t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) } @@ -3542,10 +3539,7 @@ func TestAgent_httpInjectAddr(t *testing.T) { } for _, tt := range tt { t.Run(tt.name, func(t *testing.T) { - got, err := httpInjectAddr(tt.url, tt.ip, tt.port) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + got := httpInjectAddr(tt.url, tt.ip, tt.port) if got != tt.want { t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) } From baa3a129204dd22fa0fcdbc1566f18033a70eb69 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 3 Sep 2019 20:50:45 -0600 Subject: [PATCH 33/80] Update expose config to reflect listener per path --- agent/config/builder.go | 40 ++++++++++++++++++++++++- agent/config/config.go | 43 ++++++++++++++++++++++++++- agent/config/default.go | 2 ++ agent/config/runtime.go | 8 +++++ agent/structs/connect_proxy_config.go | 33 ++++++++++---------- 5 files changed, 107 insertions(+), 19 deletions(-) diff --git a/agent/config/builder.go b/agent/config/builder.go index 45db80843da3..731f3a72a335 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -22,7 +22,7 @@ import ( "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/consul/types" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-sockaddr/template" "golang.org/x/time/rate" ) @@ -369,6 +369,8 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { proxyMaxPort := b.portVal("ports.proxy_max_port", c.Ports.ProxyMaxPort) sidecarMinPort := b.portVal("ports.sidecar_min_port", c.Ports.SidecarMinPort) sidecarMaxPort := b.portVal("ports.sidecar_max_port", c.Ports.SidecarMaxPort) + exposeMinPort := b.portVal("ports.expose_min_port", c.Ports.ExposeMinPort) + exposeMaxPort := b.portVal("ports.expose_max_port", c.Ports.ExposeMaxPort) if proxyMaxPort < proxyMinPort { return RuntimeConfig{}, fmt.Errorf( "proxy_min_port must be less than proxy_max_port. To disable, set both to zero.") @@ -377,6 +379,10 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { return RuntimeConfig{}, fmt.Errorf( "sidecar_min_port must be less than sidecar_max_port. To disable, set both to zero.") } + if exposeMaxPort < exposeMinPort { + return RuntimeConfig{}, fmt.Errorf( + "expose_min_port must be less than expose_max_port. To disable, set both to zero.") + } // determine the default bind and advertise address // @@ -804,6 +810,8 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { ConnectCAConfig: connectCAConfig, ConnectSidecarMinPort: sidecarMinPort, ConnectSidecarMaxPort: sidecarMaxPort, + ExposeMinPort: exposeMinPort, + ExposeMaxPort: exposeMaxPort, DataDir: b.stringVal(c.DataDir), Datacenter: datacenter, DevMode: b.boolVal(b.Flags.DevMode), @@ -1305,6 +1313,7 @@ func (b *Builder) serviceProxyVal(v *ServiceProxy) *structs.ConnectProxyConfig { Config: v.Config, Upstreams: b.upstreamsVal(v.Upstreams), MeshGateway: b.meshGatewayConfVal(v.MeshGateway), + Expose: b.exposeConfVal(v.Expose), } } @@ -1345,6 +1354,35 @@ func (b *Builder) meshGatewayConfVal(mgConf *MeshGatewayConfig) structs.MeshGate return cfg } +func (b *Builder) exposeConfVal(v *ExposeConfig) structs.ExposeConfig { + var out structs.ExposeConfig + if v == nil { + return out + } + + out.Checks = b.boolVal(v.Checks) + out.Paths = b.pathsVal(v.Paths) + if err := out.Finalize(); err != nil { + b.err = multierror.Append(b.err, err) + } + return out +} + +func (b *Builder) pathsVal(v []Path) []structs.Path { + paths := make([]structs.Path, len(v)) + for i, p := range v { + paths[i] = structs.Path{ + ListenerPort: b.intVal(p.ListenerPort), + Path: b.stringVal(p.Path), + LocalPathPort: b.intVal(p.LocalPathPort), + Protocol: b.stringVal(p.Protocol), + TLSSkipVerify: b.boolVal(p.TLSSkipVerify), + CAFile: b.stringVal(p.CAFile), + } + } + return paths +} + func (b *Builder) serviceConnectVal(v *ServiceConnect) *structs.ServiceConnect { if v == nil { return nil diff --git a/agent/config/config.go b/agent/config/config.go index 50deea2cb878..e423f38079a1 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/hashicorp/consul/lib" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl" "github.com/mitchellh/mapstructure" ) @@ -97,6 +97,8 @@ func Parse(data string, format string) (c Config, err error) { "services.connect.sidecar_service.checks", "service.connect.sidecar_service.proxy.upstreams", "services.connect.sidecar_service.proxy.upstreams", + "service.connect.sidecar_service.proxy.expose.paths", + "services.connect.sidecar_service.proxy.expose.paths", }, []string{ "config_entries.bootstrap", // completely ignore this tree (fixed elsewhere) }) @@ -468,6 +470,9 @@ type ServiceProxy struct { // Mesh Gateway Configuration MeshGateway *MeshGatewayConfig `json:"mesh_gateway,omitempty" hcl:"mesh_gateway" mapstructure:"mesh_gateway"` + + // Expose defines whether checks or paths are exposed through the proxy + Expose *ExposeConfig `json:"expose,omitempty" hcl:"expose" mapstructure:"expose"` } // Upstream represents a single upstream dependency for a service or proxy. It @@ -513,6 +518,40 @@ type MeshGatewayConfig struct { Mode *string `json:"mode,omitempty" hcl:"mode" mapstructure:"mode"` } +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks *bool `json:"checks,omitempty" hcl:"checks" mapstructure:"checks"` + + // Port defines the port of the proxy's listener for exposed paths. + Port *int `json:"port,omitempty" hcl:"port" mapstructure:"port"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `json:"paths,omitempty" hcl:"paths" mapstructure:"paths"` +} + +type Path struct { + // ListenerPort defines the port of the proxy's listener for exposed paths. + ListenerPort *int `json:"listener_port,omitempty" hcl:"listener_port" mapstructure:"listener_port"` + + // Path is the path to expose through the proxy, ie. "/metrics." + Path *string `json:"path,omitempty" hcl:"path" mapstructure:"path"` + + // Protocol describes the upstream's service protocol. + Protocol *string `json:"protocol,omitempty" hcl:"protocol" mapstructure:"protocol"` + + // LocalPathPort is the port that the service is listening on for the given path. + LocalPathPort *int `json:"local_path_port,omitempty" hcl:"local_path_port" mapstructure:"local_path_port"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify *bool `json:"tls_skip_verify,omitempty" hcl:"tls_skip_verify" mapstructure:"tls_skip_verify"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile *string `json:"ca_file,omitempty" hcl:"ca_file" mapstructure:"ca_file"` +} + // AutoEncrypt is the agent-global auto_encrypt configuration. type AutoEncrypt struct { // TLS enables receiving certificates for clients from servers @@ -606,6 +645,8 @@ type Ports struct { ProxyMaxPort *int `json:"proxy_max_port,omitempty" hcl:"proxy_max_port" mapstructure:"proxy_max_port"` SidecarMinPort *int `json:"sidecar_min_port,omitempty" hcl:"sidecar_min_port" mapstructure:"sidecar_min_port"` SidecarMaxPort *int `json:"sidecar_max_port,omitempty" hcl:"sidecar_max_port" mapstructure:"sidecar_max_port"` + ExposeMinPort *int `json:"expose_min_port,omitempty" hcl:"expose_min_port" mapstructure:"expose_min_port"` + ExposeMaxPort *int `json:"expose_max_port,omitempty" hcl:"expose_max_port" mapstructure:"expose_max_port"` } type UnixSocket struct { diff --git a/agent/config/default.go b/agent/config/default.go index 1580e1915250..1ceeb94ad621 100644 --- a/agent/config/default.go +++ b/agent/config/default.go @@ -122,6 +122,8 @@ func DefaultSource() Source { proxy_max_port = 20255 sidecar_min_port = 21000 sidecar_max_port = 21255 + expose_min_port = 21500 + expose_max_port = 21755 } telemetry = { metrics_prefix = "consul" diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 2d2f9b0e1d72..1b21e3ff815a 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -541,6 +541,14 @@ type RuntimeConfig struct { // specified ConnectSidecarMaxPort int + // ExposeMinPort is the inclusive start of the range of ports + // allocated to the agent for exposing checks through a proxy + ExposeMinPort int + + // ExposeMinPort is the inclusive start of the range of ports + // allocated to the agent for exposing checks through a proxy + ExposeMaxPort int + // ConnectCAProvider is the type of CA provider to use with Connect. ConnectCAProvider string diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index a40bc78ccf02..a93220126023 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -11,11 +11,10 @@ import ( ) const ( - defaultExposePort = 21500 defaultExposeProtocol = "http" ) -var allowedExposeProtocols = map[string]bool{"http": true, "http2": true, "grpc": true} +var allowedExposeProtocols = map[string]bool{"http": true, "http2": true} type MeshGatewayMode string @@ -332,22 +331,22 @@ type ExposeConfig struct { // This flag triggers exposing all HTTP and GRPC check paths registered for the service. Checks bool `mapstructure:"checks"` - // Port defines the port of the proxy's listener for exposed paths. - Port int `mapstructure:"port"` - // Paths is the list of paths exposed through the proxy. Paths []Path `mapstructure:"paths"` } type Path struct { + // ListenerPort defines the port of the proxy's listener for exposed paths. + ListenerPort int `mapstructure:"listener_port"` + // Path is the path to expose through the proxy, ie. "/metrics." Path string `mapstructure:"path"` - // Port is the port that the service is listening on for the given path. - Port int `mapstructure:"port"` + // LocalPathPort is the port that the service is listening on for the given path. + LocalPathPort int `mapstructure:"local_path_port"` // Protocol describes the upstream's service protocol. - // Valid values are "http", "http2" and "grpc". Defaults to "http". + // Valid values are "http" and "http2", defaults to "http" Protocol string `mapstructure:"protocol"` // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. @@ -362,20 +361,19 @@ type Path struct { // Finalize validates ExposeConfig and sets default values func (e *ExposeConfig) Finalize() error { - if e.Port < 0 || e.Port > 65535 { - return fmt.Errorf("invalid port: %d", e.Port) - } - if e.Port == 0 { - e.Port = defaultExposePort - } - var known = make(map[string]bool) - for _, path := range e.Paths { + for i := 0; i < len(e.Paths); i++ { + path := &e.Paths[i] + if seen := known[path.Path]; seen { return fmt.Errorf("duplicate paths exposed") } known[path.Path] = true + if path.ListenerPort <= 0 || path.ListenerPort > 65535 { + return fmt.Errorf("invalid port: %d", path.ListenerPort) + } + if path.CAFile != "" { b, err := ioutil.ReadFile(path.CAFile) if err != nil { @@ -386,7 +384,8 @@ func (e *ExposeConfig) Finalize() error { path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { - return fmt.Errorf("protocol '%s' not recognized for path: %s", path.Protocol, path.Path) + return fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", + path.Protocol, path.Path) } if path.Protocol == "" { path.Protocol = defaultExposeProtocol From 859bb5bfbb0f451cf1fc12ccfd3c2c8a028edc7e Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 3 Sep 2019 20:56:54 -0600 Subject: [PATCH 34/80] Create Envoy listeners and clusters for exposed paths --- agent/xds/clusters.go | 31 +++++++--- agent/xds/listeners.go | 131 +++++++++++++++++++++++++++++++---------- 2 files changed, 125 insertions(+), 37 deletions(-) diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 69d3be65a8e4..a354a5d11ee8 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -44,7 +44,7 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh clusters := make([]proto.Message, 0, len(cfgSnap.Proxy.Upstreams)+1) // Include the "app" cluster for the public listener - appCluster, err := s.makeAppCluster(cfgSnap) + appCluster, err := s.makeAppCluster(cfgSnap, LocalAppClusterName, "", cfgSnap.Proxy.LocalServicePort) if err != nil { return nil, err } @@ -74,9 +74,26 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } } + // Create a new cluster if we need to expose a port that is different from the service port + for _, path := range cfgSnap.Proxy.Expose.Paths { + if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { + continue + } + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path), path.Protocol, path.LocalPathPort) + if err != nil { + s.Logger.Printf("[WARN] envoy: failed to make local cluster for '%s': %s", path.Path, err) + continue + } + clusters = append(clusters, c) + } + return clusters, nil } +func makeExposeClusterName(path structs.Path) string { + return fmt.Sprintf("exposed_cluster_%d", path.LocalPathPort) +} + // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters" // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. @@ -122,7 +139,7 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho return clusters, nil } -func (s *Server) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot) (*envoy.Cluster, error) { +func (s *Server) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathProtocol string, port int) (*envoy.Cluster, error) { var c *envoy.Cluster var err error @@ -143,23 +160,23 @@ func (s *Server) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot) (*envoy.Cluste addr = "127.0.0.1" } c = &envoy.Cluster{ - Name: LocalAppClusterName, + Name: name, ConnectTimeout: time.Duration(cfg.LocalConnectTimeoutMs) * time.Millisecond, ClusterDiscoveryType: &envoy.Cluster_Type{Type: envoy.Cluster_STATIC}, LoadAssignment: &envoy.ClusterLoadAssignment{ - ClusterName: LocalAppClusterName, + ClusterName: name, Endpoints: []envoyendpoint.LocalityLbEndpoints{ { LbEndpoints: []envoyendpoint.LbEndpoint{ - makeEndpoint(LocalAppClusterName, + makeEndpoint(name, addr, - cfgSnap.Proxy.LocalServicePort), + port), }, }, }, }, } - if cfg.Protocol == "http2" || cfg.Protocol == "grpc" { + if cfg.Protocol == "http2" || cfg.Protocol == "grpc" || pathProtocol == "http2" { c.Http2ProtocolOptions = &envoycore.Http2ProtocolOptions{} } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 9c635e1056ec..dc5e9e787503 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "strings" envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" @@ -71,6 +72,16 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps } resources[i+1] = upstreamListener } + + // Configure additional listener for exposed check paths + for _, path := range cfgSnap.Proxy.Expose.Paths { + l, err := s.makeExposedCheckListener(cfgSnap, path) + if err != nil { + return nil, err + } + resources = append(resources, l) + } + return resources, nil } @@ -253,7 +264,8 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri l = makeListener(PublicListenerName, addr, port) - filter, err := makeListenerFilter(false, cfg.Protocol, "public_listener", LocalAppClusterName, "", true) + filter, err := makeListenerFilter( + false, cfg.Protocol, "public_listener", LocalAppClusterName, "", "", true) if err != nil { return nil, err } @@ -270,6 +282,49 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri return l, err } +func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, path structs.Path) (proto.Message, error) { + cfg, err := ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + } + + // No user config, use default listener + addr := cfgSnap.Address + + // Override with bind address if one is set, otherwise default to 0.0.0.0 + if cfg.BindAddress != "" { + addr = cfg.BindAddress + } else if addr == "" { + addr = "0.0.0.0" + } + + // Strip any special characters from path + r := regexp.MustCompile(`[^a-zA-Z0-9]+`) + strippedPath := r.ReplaceAllString(path.Path, "") + listenerName := fmt.Sprintf("exposed_path_listener_%s_%d", strippedPath, path.ListenerPort) + l := makeListener(listenerName, addr, path.ListenerPort) + + filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, path.ListenerPort) + clusterName := LocalAppClusterName + if path.LocalPathPort != cfgSnap.Proxy.LocalServicePort { + clusterName = makeExposeClusterName(path) + } + + f, err := makeListenerFilter(false, path.Protocol, filterName, clusterName, "", path.Path, true) + if err != nil { + return nil, err + } + + l.FilterChains = []envoylistener.FilterChain{ + { + Filters: []envoylistener.Filter{f}, + }, + } + return l, err +} + // makeUpstreamListenerIgnoreDiscoveryChain counterintuitively takes an (optional) chain func (s *Server) makeUpstreamListenerIgnoreDiscoveryChain( u *structs.Upstream, @@ -303,7 +358,8 @@ func (s *Server) makeUpstreamListenerIgnoreDiscoveryChain( clusterName := CustomizeClusterName(sni, chain) l := makeListener(upstreamID, addr, u.LocalBindPort) - filter, err := makeListenerFilter(false, cfg.Protocol, upstreamID, clusterName, "upstream_", false) + filter, err := makeListenerFilter( + false, cfg.Protocol, upstreamID, clusterName, "upstream_", "", false) if err != nil { return nil, err } @@ -408,7 +464,8 @@ func (s *Server) makeUpstreamListenerForDiscoveryChain( proto = "tcp" } - filter, err := makeListenerFilter(true, proto, upstreamID, "", "upstream_", false) + filter, err := makeListenerFilter( + true, proto, upstreamID, "", "upstream_", "", false) if err != nil { return nil, err } @@ -423,14 +480,17 @@ func (s *Server) makeUpstreamListenerForDiscoveryChain( return l, nil } -func makeListenerFilter(useRDS bool, protocol, filterName, cluster, statPrefix string, ingress bool) (envoylistener.Filter, error) { +func makeListenerFilter( + useRDS bool, + protocol, filterName, cluster, statPrefix, routePath string, ingress bool) (envoylistener.Filter, error) { + switch protocol { case "grpc": - return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, ingress, true, true) + return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, routePath, ingress, true, true) case "http2": - return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, ingress, false, true) + return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, routePath, ingress, false, true) case "http": - return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, ingress, false, false) + return makeHTTPFilter(useRDS, filterName, cluster, statPrefix, routePath, ingress, false, false) case "tcp": fallthrough default: @@ -471,7 +531,7 @@ func makeStatPrefix(protocol, prefix, filterName string) string { func makeHTTPFilter( useRDS bool, - filterName, cluster, statPrefix string, + filterName, cluster, statPrefix, routePath string, ingress, grpc, http2 bool, ) (envoylistener.Filter, error) { op := envoyhttp.INGRESS @@ -482,9 +542,14 @@ func makeHTTPFilter( if grpc { proto = "grpc" } + codec := envoyhttp.AUTO + if grpc || http2 { + codec = envoyhttp.HTTP2 + } + cfg := &envoyhttp.HttpConnectionManager{ StatPrefix: makeStatPrefix(proto, statPrefix, filterName), - CodecType: envoyhttp.AUTO, + CodecType: codec, HttpFilters: []*envoyhttp.HttpFilter{ &envoyhttp.HttpFilter{ Name: "envoy.router", @@ -517,33 +582,39 @@ func makeHTTPFilter( if cluster == "" { return envoylistener.Filter{}, fmt.Errorf("must specify cluster name when not using RDS") } + route := envoyroute.Route{ + Match: envoyroute.RouteMatch{ + PathSpecifier: &envoyroute.RouteMatch_Prefix{ + Prefix: "/", + }, + // TODO(banks) Envoy supports matching only valid GRPC + // requests which might be nice to add here for gRPC services + // but it's not supported in our current envoy SDK version + // although docs say it was supported by 1.8.0. Going to defer + // that until we've updated the deps. + }, + Action: &envoyroute.Route_Route{ + Route: &envoyroute.RouteAction{ + ClusterSpecifier: &envoyroute.RouteAction_Cluster{ + Cluster: cluster, + }, + }, + }, + } + // If a path is provided, do not match on a catch-all prefix + if routePath != "" { + route.Match.PathSpecifier = &envoyroute.RouteMatch_Path{Path: routePath} + } + cfg.RouteSpecifier = &envoyhttp.HttpConnectionManager_RouteConfig{ RouteConfig: &envoy.RouteConfiguration{ Name: filterName, VirtualHosts: []envoyroute.VirtualHost{ - envoyroute.VirtualHost{ + { Name: filterName, Domains: []string{"*"}, Routes: []envoyroute.Route{ - envoyroute.Route{ - Match: envoyroute.RouteMatch{ - PathSpecifier: &envoyroute.RouteMatch_Prefix{ - Prefix: "/", - }, - // TODO(banks) Envoy supports matching only valid GRPC - // requests which might be nice to add here for gRPC services - // but it's not supported in our current envoy SDK version - // although docs say it was supported by 1.8.0. Going to defer - // that until we've updated the deps. - }, - Action: &envoyroute.Route_Route{ - Route: &envoyroute.RouteAction{ - ClusterSpecifier: &envoyroute.RouteAction_Cluster{ - Cluster: cluster, - }, - }, - }, - }, + route, }, }, }, @@ -557,7 +628,7 @@ func makeHTTPFilter( if grpc { // Add grpc bridge before router - cfg.HttpFilters = append([]*envoyhttp.HttpFilter{&envoyhttp.HttpFilter{ + cfg.HttpFilters = append([]*envoyhttp.HttpFilter{{ Name: "envoy.grpc_http1_bridge", ConfigType: &envoyhttp.HttpFilter_Config{Config: &types.Struct{}}, }}, cfg.HttpFilters...) From 17cf99699ad70211d411efceca52811d4ad8ed4b Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 14:04:10 -0600 Subject: [PATCH 35/80] Update configuration and add cfg tests --- agent/config/config.go | 4 + agent/config/runtime_test.go | 141 +++++++++++++++++- agent/structs/check_type.go | 4 + agent/structs/connect_proxy_config.go | 4 +- agent/structs/connect_proxy_config_test.go | 158 +++++++++++++++++++++ 5 files changed, 306 insertions(+), 5 deletions(-) diff --git a/agent/config/config.go b/agent/config/config.go index e423f38079a1..4f4d53a71db0 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -89,8 +89,12 @@ func Parse(data string, format string) (c Config, err error) { "services.connect.proxy.config.upstreams", // Deprecated "service.connect.proxy.upstreams", "services.connect.proxy.upstreams", + "service.connect.proxy.expose.paths", + "services.connect.proxy.expose.paths", "service.proxy.upstreams", "services.proxy.upstreams", + "service.proxy.expose.paths", + "services.proxy.expose.paths", // Need all the service(s) exceptions also for nested sidecar service. "service.connect.sidecar_service.checks", diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index a114132f91a3..ba7526c4af26 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -1295,6 +1295,36 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { rt.DataDir = dataDir }, }, + { + desc: "min/max ports for dynamic exposed listeners", + args: []string{`-data-dir=` + dataDir}, + json: []string{`{ + "ports": { + "expose_min_port": 1234, + "expose_max_port": 5678 + } + }`}, + hcl: []string{` + ports { + expose_min_port = 1234 + expose_max_port = 5678 + } + `}, + patch: func(rt *RuntimeConfig) { + rt.ExposeMinPort = 1234 + rt.ExposeMaxPort = 5678 + rt.DataDir = dataDir + }, + }, + { + desc: "defaults for dynamic exposed listeners", + args: []string{`-data-dir=` + dataDir}, + patch: func(rt *RuntimeConfig) { + rt.ExposeMinPort = 21500 + rt.ExposeMaxPort = 21755 + rt.DataDir = dataDir + }, + }, // ------------------------------------------------------------ // precedence rules @@ -2338,6 +2368,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ], "proxy": { + "expose": { + "checks": true, + "paths": [ + { + "path": "/health", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http" + } + ] + }, "upstreams": [ { "destination_name": "db", @@ -2363,6 +2404,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ] proxy { + expose { + checks = true + paths = [ + { + path = "/health" + local_path_port = 8080 + listener_port = 21500 + protocol = "http" + } + ] + }, upstreams = [ { destination_name = "db" @@ -2391,6 +2443,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { }, }, Proxy: &structs.ConnectProxyConfig{ + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.Path{ + { + Path: "/health", + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http", + }, + }, + }, Upstreams: structs.Upstreams{ structs.Upstream{ DestinationType: "service", @@ -2434,6 +2497,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ], "proxy": { + "expose": { + "checks": true, + "paths": [ + { + "path": "/health", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http" + } + ] + }, "upstreams": [ { "destination_name": "db", @@ -2459,6 +2533,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { } ] proxy { + expose { + checks = true + paths = [ + { + path = "/health" + local_path_port = 8080 + listener_port = 21500 + protocol = "http" + } + ] + }, upstreams = [ { destination_name = "db" @@ -2487,6 +2572,17 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { }, }, Proxy: &structs.ConnectProxyConfig{ + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.Path{ + { + Path: "/health", + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http", + }, + }, + }, Upstreams: structs.Upstreams{ structs.Upstream{ DestinationType: "service", @@ -3627,7 +3723,9 @@ func TestFullConfig(t *testing.T) { "server": 3757, "grpc": 4881, "sidecar_min_port": 8888, - "sidecar_max_port": 9999 + "sidecar_max_port": 9999, + "expose_min_port": 1111, + "expose_max_port": 2222 }, "protocol": 30793, "primary_datacenter": "ejtmd43d", @@ -3871,6 +3969,17 @@ func TestFullConfig(t *testing.T) { "destination_service_name": "6L6BVfgH", "local_service_address": "127.0.0.2", "local_service_port": 23759, + "expose": { + "checks": true, + "paths": [ + { + "path": "/health", + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http" + } + ] + }, "upstreams": [ { "destination_name": "KPtAj2cb", @@ -4205,8 +4314,8 @@ func TestFullConfig(t *testing.T) { } pid_file = "43xN80Km" ports { - dns = 7001, - http = 7999, + dns = 7001 + http = 7999 https = 15127 server = 3757 grpc = 4881 @@ -4214,6 +4323,8 @@ func TestFullConfig(t *testing.T) { proxy_max_port = 3000 sidecar_min_port = 8888 sidecar_max_port = 9999 + expose_min_port = 1111 + expose_max_port = 2222 } protocol = 30793 primary_datacenter = "ejtmd43d" @@ -4472,6 +4583,17 @@ func TestFullConfig(t *testing.T) { local_bind_address = "127.24.88.0" }, ] + expose { + checks = true + paths = [ + { + path = "/health" + local_path_port = 8080 + listener_port = 21500 + protocol = "http" + } + ] + } } }, { @@ -4797,6 +4919,8 @@ func TestFullConfig(t *testing.T) { ConnectEnabled: true, ConnectSidecarMinPort: 8888, ConnectSidecarMaxPort: 9999, + ExposeMinPort: 1111, + ExposeMaxPort: 2222, ConnectCAProvider: "consul", ConnectCAConfig: map[string]interface{}{ "RotationPeriod": "90h", @@ -5044,6 +5168,17 @@ func TestFullConfig(t *testing.T) { LocalBindAddress: "127.24.88.0", }, }, + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.Path{ + { + Path: "/health", + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http", + }, + }, + }, }, Weights: &structs.Weights{ Passing: 1, diff --git a/agent/structs/check_type.go b/agent/structs/check_type.go index 9b1b055dae30..1b217e2e91ab 100644 --- a/agent/structs/check_type.go +++ b/agent/structs/check_type.go @@ -41,6 +41,10 @@ type CheckType struct { Timeout time.Duration TTL time.Duration + // Definition fields used when exposing checks through a proxy + ProxyHTTP string + ProxyGRPC string + // DeregisterCriticalServiceAfter, if >0, will cause the associated // service, if any, to be deregistered if this check is critical for // longer than this duration. diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index a93220126023..db9260070c10 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -371,13 +371,13 @@ func (e *ExposeConfig) Finalize() error { known[path.Path] = true if path.ListenerPort <= 0 || path.ListenerPort > 65535 { - return fmt.Errorf("invalid port: %d", path.ListenerPort) + return fmt.Errorf("invalid listener port: %d", path.ListenerPort) } if path.CAFile != "" { b, err := ioutil.ReadFile(path.CAFile) if err != nil { - return fmt.Errorf("failed to read '%s': %v", path.CAFile, err) + return fmt.Errorf("failed to read CAFile '%s': %v", path.CAFile, err) } path.CACert = string(b) } diff --git a/agent/structs/connect_proxy_config_test.go b/agent/structs/connect_proxy_config_test.go index bc1dd0f50d17..6326ab253c33 100644 --- a/agent/structs/connect_proxy_config_test.go +++ b/agent/structs/connect_proxy_config_test.go @@ -3,6 +3,9 @@ package structs import ( "encoding/json" "fmt" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" "testing" "github.com/hashicorp/consul/api" @@ -270,3 +273,158 @@ func TestValidateMeshGatewayMode(t *testing.T) { }) } } + +func TestExposeConfig_Finalize(t *testing.T) { + t.Parallel() + + tmpDir, err := ioutil.TempDir("", "exposeconfig_") + if err != nil { + t.Fatalf("failed to create tempdir: %v", err) + } + defer os.RemoveAll(tmpDir) + + tmpFile, err := ioutil.TempFile(tmpDir, "CAFile") + if err != nil { + t.Fatalf("failed to create tempfile: %v", err) + } + defer tmpFile.Close() + + type fields struct { + Checks bool + Paths []Path + } + tests := []struct { + name string + fields fields + want ExposeConfig + wantErr bool + errMsg string + }{ + { + name: "duplicate path", + fields: fields{ + Paths: []Path{ + { + LocalPathPort: 80, + ListenerPort: 80, + Path: "/metrics", + }, + { + LocalPathPort: 80, + ListenerPort: 80, + Path: "/metrics", + }, + }, + }, + wantErr: true, + errMsg: "duplicate paths exposed", + }, + { + name: "negative listener port", + fields: fields{ + Paths: []Path{ + { + ListenerPort: -1, + }, + }, + }, + wantErr: true, + errMsg: "invalid listener port: -1", + }, + { + name: "listener port too large", + fields: fields{ + Paths: []Path{ + { + ListenerPort: 65536, + }, + }, + }, + wantErr: true, + errMsg: "invalid listener port: 65536", + }, + { + name: "protocol not supported", + fields: fields{ + Paths: []Path{ + { + Path: "/metrics", + LocalPathPort: 80, + ListenerPort: 80, + Protocol: "tcp", + }, + }, + }, + wantErr: true, + errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", + }, + { + name: "protocol not supported", + fields: fields{ + Paths: []Path{ + { + Path: "/metrics", + LocalPathPort: 80, + ListenerPort: 80, + Protocol: "tcp", + }, + }, + }, + wantErr: true, + errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", + }, + { + name: "default to http when no protocol", + fields: fields{ + Paths: []Path{ + { + LocalPathPort: 80, + ListenerPort: 80, + }, + }, + }, + wantErr: false, + want: ExposeConfig{ + Paths: []Path{ + {LocalPathPort: 80, ListenerPort: 80, Protocol: "http"}, + }, + }, + }, + { + name: "lowercase protocol", + fields: fields{ + Paths: []Path{ + { + LocalPathPort: 80, + ListenerPort: 80, + Protocol: "HTTP2", + }, + }, + }, + wantErr: false, + want: ExposeConfig{ + Paths: []Path{ + {LocalPathPort: 80, ListenerPort: 80, Protocol: "http2"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &ExposeConfig{ + Checks: tt.fields.Checks, + Paths: tt.fields.Paths, + } + err := e.Finalize() + if (err != nil) != tt.wantErr { + t.Errorf("Finalize() got error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr && tt.errMsg != err.Error() { + t.Errorf("Finalize() got error: '%v', want: '%s'", err, tt.errMsg) + } + if !tt.wantErr { + assert.Equal(t, &tt.want, e) + } + }) + } +} From 931d167ebb2300839b218d08871f22323c60175d Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 14:04:50 -0600 Subject: [PATCH 36/80] Create clusters and listeners for expose Consul checks --- agent/agent.go | 11 ++--- agent/checks/check.go | 10 +++-- agent/xds/clusters.go | 23 +++++++--- agent/xds/listeners.go | 98 +++++++++++++++++++++++++++++++++++------- agent/xds/server.go | 7 +++ 5 files changed, 120 insertions(+), 29 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 56edcbae58ce..2171feda99e1 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -670,6 +670,7 @@ func (a *Agent) listenAndServeGRPC() error { CfgMgr: a.proxyConfig, Authz: a, ResolveToken: a.resolveToken, + CheckFetcher: a, } a.xdsServer.Initialize() @@ -3631,6 +3632,7 @@ func (a *Agent) resetExposedChecks(serviceID string) { } // listenerPort allocates a port from the configured range +// The agent stateLock MUST be held when this is called func (a *Agent) listenerPort(svcID, checkID string) (int, error) { key := fmt.Sprintf("%s:%s", svcID, checkID) if a.exposedPorts == nil { @@ -3646,9 +3648,10 @@ func (a *Agent) listenerPort(svcID, checkID string) (int, error) { } var port int - for i := a.config.ExposeMinPort; i < a.config.ExposeMaxPort; i++ { + for i := 0; i < a.config.ExposeMaxPort-a.config.ExposeMinPort; i++ { port = a.config.ExposeMinPort + i if !allocated[port] { + a.exposedPorts[key] = port break } } @@ -3661,8 +3664,7 @@ func (a *Agent) listenerPort(svcID, checkID string) (int, error) { // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] func grpcInjectAddr(existing string, ip string, port int) string { - pattern := "(.*)((?::)(?:[0-9]+))(.*)$" - r := regexp.MustCompile(pattern) + r := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") portRepl := fmt.Sprintf("${1}:%d${3}", port) out := r.ReplaceAllString(existing, portRepl) @@ -3675,8 +3677,7 @@ func grpcInjectAddr(existing string, ip string, port int) string { // httpInjectAddr injects a port then an IP into a URL func httpInjectAddr(url string, ip string, port int) string { - pattern := "^(http[s]?://)(\\[.*?\\]|\\[?[\\w\\-\\.]+)(:\\d+)?([^?]*)(\\?.*)?$" - r := regexp.MustCompile(pattern) + r := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) out := r.ReplaceAllString(url, portRepl) diff --git a/agent/checks/check.go b/agent/checks/check.go index d25566ff7ae2..bb4ca538e99a 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -338,6 +338,7 @@ func (c *CheckHTTP) CheckType() structs.CheckType { Method: c.Method, Header: c.Header, Interval: c.Interval, + ProxyHTTP: c.ProxyHTTP, Timeout: c.Timeout, OutputMaxSize: c.OutputMaxSize, } @@ -702,10 +703,11 @@ type CheckGRPC struct { func (c *CheckGRPC) CheckType() structs.CheckType { return structs.CheckType{ - CheckID: c.CheckID, - GRPC: c.GRPC, - Interval: c.Interval, - Timeout: c.Timeout, + CheckID: c.CheckID, + GRPC: c.GRPC, + ProxyGRPC: c.ProxyGRPC, + Interval: c.Interval, + Timeout: c.Timeout, } } diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index a354a5d11ee8..70b2fa87a5fb 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -74,24 +74,37 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } } + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create clusters for if needed + if cfgSnap.Proxy.Expose.Checks { + for _, check := range s.CheckFetcher.ServiceHTTPChecks(cfgSnap.Proxy.DestinationServiceID) { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Printf("[WARN] envoy: failed to create cluster for check '%s': %v", check.CheckID, err) + continue + } + paths = append(paths, p) + } + } + // Create a new cluster if we need to expose a port that is different from the service port - for _, path := range cfgSnap.Proxy.Expose.Paths { + for _, path := range paths { if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { continue } - c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path), path.Protocol, path.LocalPathPort) + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path.LocalPathPort), path.Protocol, path.LocalPathPort) if err != nil { s.Logger.Printf("[WARN] envoy: failed to make local cluster for '%s': %s", path.Path, err) continue } clusters = append(clusters, c) } - return clusters, nil } -func makeExposeClusterName(path structs.Path) string { - return fmt.Sprintf("exposed_cluster_%d", path.LocalPathPort) +func makeExposeClusterName(destinationPort int) string { + return fmt.Sprintf("exposed_cluster_%d", destinationPort) } // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters" diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index dc5e9e787503..28269b5216c0 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "regexp" + "strconv" "strings" envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" @@ -73,9 +74,28 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps resources[i+1] = upstreamListener } + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create listeners for if needed + if cfgSnap.Proxy.Expose.Checks { + for _, check := range s.CheckFetcher.ServiceHTTPChecks(cfgSnap.Proxy.DestinationServiceID) { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Printf("[WARN] envoy: failed to create listener for check '%s': %v", check.CheckID, err) + continue + } + paths = append(paths, p) + } + } + // Configure additional listener for exposed check paths - for _, path := range cfgSnap.Proxy.Expose.Paths { - l, err := s.makeExposedCheckListener(cfgSnap, path) + for _, path := range paths { + clusterName := LocalAppClusterName + if path.LocalPathPort != cfgSnap.Proxy.LocalServicePort { + clusterName = makeExposeClusterName(path.LocalPathPort) + } + + l, err := s.makeExposedCheckListener(cfgSnap, clusterName, path.Path, path.Protocol, path.ListenerPort) if err != nil { return nil, err } @@ -85,13 +105,64 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps return resources, nil } +func parseCheckPath(check structs.CheckType) (structs.Path, error) { + grpcRE := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") + httpRE := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) + + var path structs.Path + var err error + + if check.HTTP != "" { + path.Protocol = "http" + + matches := httpRE.FindStringSubmatch(check.HTTP) + path.Path = matches[4] + + localStr := strings.TrimPrefix(matches[3], ":") + path.LocalPathPort, err = strconv.Atoi(localStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.HTTP, err) + } + + matches = httpRE.FindStringSubmatch(check.ProxyHTTP) + + listenerStr := strings.TrimPrefix(matches[3], ":") + path.ListenerPort, err = strconv.Atoi(listenerStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyHTTP, err) + } + } + + if check.GRPC != "" { + path.Path = "/grpc.health.v1.Health/Check" + path.Protocol = "http2" + + matches := grpcRE.FindStringSubmatch(check.GRPC) + + localStr := strings.TrimPrefix(matches[2], ":") + path.LocalPathPort, err = strconv.Atoi(localStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.GRPC, err) + } + + matches = grpcRE.FindStringSubmatch(check.ProxyGRPC) + + listenerStr := strings.TrimPrefix(matches[2], ":") + path.ListenerPort, err = strconv.Atoi(listenerStr) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyGRPC, err) + } + } + return path, nil +} + // listenersFromSnapshotMeshGateway returns the "listener" for a mesh-gateway service func (s *Server) listenersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) { cfg, err := ParseMeshGatewayConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. - s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %v", err) } // TODO - prevent invalid configurations of binding to the same port/addr @@ -232,7 +303,7 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. - s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %v", err) } if cfg.PublicListenerJSON != "" { @@ -282,12 +353,12 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri return l, err } -func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, path structs.Path) (proto.Message, error) { +func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, cluster, path, protocol string, port int) (proto.Message, error) { cfg, err := ParseProxyConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. - s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %s", err) + s.Logger.Printf("[WARN] envoy: failed to parse Connect.Proxy.Config: %v", err) } // No user config, use default listener @@ -302,17 +373,14 @@ func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, path // Strip any special characters from path r := regexp.MustCompile(`[^a-zA-Z0-9]+`) - strippedPath := r.ReplaceAllString(path.Path, "") - listenerName := fmt.Sprintf("exposed_path_listener_%s_%d", strippedPath, path.ListenerPort) - l := makeListener(listenerName, addr, path.ListenerPort) + strippedPath := r.ReplaceAllString(path, "") + listenerName := fmt.Sprintf("exposed_path_%s", strippedPath) - filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, path.ListenerPort) - clusterName := LocalAppClusterName - if path.LocalPathPort != cfgSnap.Proxy.LocalServicePort { - clusterName = makeExposeClusterName(path) - } + l := makeListener(listenerName, addr, port) + + filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, port) - f, err := makeListenerFilter(false, path.Protocol, filterName, clusterName, "", path.Path, true) + f, err := makeListenerFilter(false, protocol, filterName, cluster, "", path, true) if err != nil { return nil, err } diff --git a/agent/xds/server.go b/agent/xds/server.go index f4ee9fa15c7b..1cc346dba044 100644 --- a/agent/xds/server.go +++ b/agent/xds/server.go @@ -97,6 +97,12 @@ type ConnectAuthz interface { ConnectAuthorize(token string, req *structs.ConnectAuthorizeRequest) (authz bool, reason string, m *cache.ResultMeta, err error) } +// ServiceChecks is the interface the agent needs to expose +// for the xDS server to fetch a service's HTTP check definitions +type HTTPCheckFetcher interface { + ServiceHTTPChecks(serviceID string) []structs.CheckType +} + // ConfigManager is the interface xds.Server requires to consume proxy config // updates. It's satisfied normally by the agent's proxycfg.Manager, but allows // easier testing without several layers of mocked cache, local state and @@ -121,6 +127,7 @@ type Server struct { // This is only used during idle periods of stream interactions (i.e. when // there has been no recent DiscoveryRequest). AuthCheckFrequency time.Duration + CheckFetcher HTTPCheckFetcher } // Initialize will finish configuring the Server for first use. From 48f7ddf4eb531bf26b92a05f2025ec99d7a62b1d Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 14:14:20 -0600 Subject: [PATCH 37/80] Fix grpc test --- agent/checks/grpc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/checks/grpc_test.go b/agent/checks/grpc_test.go index 3c86093b0e70..ebfe8aa518dc 100644 --- a/agent/checks/grpc_test.go +++ b/agent/checks/grpc_test.go @@ -88,7 +88,7 @@ func TestCheck(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { probe := NewGrpcHealthProbe(tt.args.target, tt.args.timeout, tt.args.tlsConfig) - actualError := probe.Check() + actualError := probe.Check(tt.args.target) actuallyHealthy := actualError == nil if tt.healthy != actuallyHealthy { t.Errorf("FAIL: %s. Expected healthy %t, but err == %v", tt.name, tt.healthy, actualError) From ada090d96fd72097489b0daa13a6aa2bc8b78d84 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 16:00:15 -0600 Subject: [PATCH 38/80] Fix broken tests --- agent/agent_endpoint.go | 7 ++- agent/config/runtime_test.go | 4 ++ agent/structs/service_definition.go | 1 + agent/structs/structs.go | 2 +- agent/structs/structs_filtering_test.go | 55 +++++++++++++++++++ ...nect-proxy-with-chain-and-overrides.golden | 1 + .../connect-proxy-with-grpc-chain.golden | 1 + .../connect-proxy-with-http2-chain.golden | 1 + api/agent.go | 1 + api/config_entry.go | 35 ++++++++++++ 10 files changed, 106 insertions(+), 2 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index c7a887e465b7..78a8df6a69d2 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -24,7 +24,7 @@ import ( "github.com/hashicorp/consul/lib/file" "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/types" - bexpr "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/go-bexpr" "github.com/hashicorp/logutils" "github.com/hashicorp/serf/coordinate" "github.com/hashicorp/serf/serf" @@ -769,6 +769,11 @@ func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Re "local_service_address": "LocalServiceAddress", // SidecarService "sidecar_service": "SidecarService", + // Expose Config + "local_path_port": "LocalPathPort", + "listener_port": "ListenerPort", + "tls_skip_verify": "TLSSkipVerify", + "ca_file": "CAFile", // DON'T Recurse into these opaque config maps or we might mangle user's // keys. Note empty canonical is a special sentinel to prevent recursion. diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index ba7526c4af26..60b6a1a62d30 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -5818,6 +5818,8 @@ func TestSanitize(t *testing.T) { "EncryptKey": "hidden", "EncryptVerifyIncoming": false, "EncryptVerifyOutgoing": false, + "ExposeMaxPort": 0, + "ExposeMinPort": 0, "GRPCAddrs": [], "GRPCPort": 0, "HTTPAddrs": [ @@ -5898,6 +5900,8 @@ func TestSanitize(t *testing.T) { "Name": "blurb", "Notes": "", "OutputMaxSize": ` + strconv.Itoa(checks.DefaultBufSize) + `, + "ProxyGRPC": "", + "ProxyHTTP": "", "ScriptArgs": [], "Shell": "", "Status": "", diff --git a/agent/structs/service_definition.go b/agent/structs/service_definition.go index 26b296887d2d..0ba6c6c5eb9f 100644 --- a/agent/structs/service_definition.go +++ b/agent/structs/service_definition.go @@ -54,6 +54,7 @@ func (s *ServiceDefinition) NodeService() *NodeService { ns.Proxy.Upstreams[i].DestinationType = UpstreamDestTypeService } } + ns.Proxy.Expose = s.Proxy.Expose } if ns.ID == "" && ns.Service != "" { ns.ID = ns.Service diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 1676b85e7085..7880fef0b486 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -14,7 +14,7 @@ import ( "time" "github.com/hashicorp/go-msgpack/codec" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/serf/coordinate" "github.com/mitchellh/hashstructure" diff --git a/agent/structs/structs_filtering_test.go b/agent/structs/structs_filtering_test.go index ffd9342f209b..6b3ae2091387 100644 --- a/agent/structs/structs_filtering_test.go +++ b/agent/structs/structs_filtering_test.go @@ -60,6 +60,57 @@ var expectedFieldConfigMeshGatewayConfig bexpr.FieldConfigurations = bexpr.Field }, } +var expectedFieldConfigExposeConfig bexpr.FieldConfigurations = bexpr.FieldConfigurations{ + "Checks": &bexpr.FieldConfiguration{ + StructFieldName: "Checks", + CoerceFn: bexpr.CoerceBool, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Paths": &bexpr.FieldConfiguration{ + StructFieldName: "Paths", + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchIsEmpty, bexpr.MatchIsNotEmpty}, + SubFields: expectedFieldConfigPaths, + }, +} + +var expectedFieldConfigPaths bexpr.FieldConfigurations = bexpr.FieldConfigurations{ + "ListenerPort": &bexpr.FieldConfiguration{ + StructFieldName: "ListenerPort", + CoerceFn: bexpr.CoerceInt, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Path": &bexpr.FieldConfiguration{ + StructFieldName: "Path", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "LocalPathPort": &bexpr.FieldConfiguration{ + StructFieldName: "LocalPathPort", + CoerceFn: bexpr.CoerceInt, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Protocol": &bexpr.FieldConfiguration{ + StructFieldName: "Protocol", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "TLSSkipVerify": &bexpr.FieldConfiguration{ + StructFieldName: "TLSSkipVerify", + CoerceFn: bexpr.CoerceBool, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "CAFile": &bexpr.FieldConfiguration{ + StructFieldName: "CAFile", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "CACert": &bexpr.FieldConfiguration{ + StructFieldName: "CACert", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, +} + var expectedFieldConfigUpstreams bexpr.FieldConfigurations = bexpr.FieldConfigurations{ "DestinationType": &bexpr.FieldConfiguration{ StructFieldName: "DestinationType", @@ -127,6 +178,10 @@ var expectedFieldConfigConnectProxyConfig bexpr.FieldConfigurations = bexpr.Fiel StructFieldName: "MeshGateway", SubFields: expectedFieldConfigMeshGatewayConfig, }, + "Expose": &bexpr.FieldConfiguration{ + StructFieldName: "Expose", + SubFields: expectedFieldConfigExposeConfig, + }, } var expectedFieldConfigServiceConnect bexpr.FieldConfigurations = bexpr.FieldConfigurations{ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden index a911cd1890b3..41269e441840 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden @@ -16,6 +16,7 @@ { "name": "envoy.http_connection_manager", "config": { + "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden b/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden index a911cd1890b3..41269e441840 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden @@ -16,6 +16,7 @@ { "name": "envoy.http_connection_manager", "config": { + "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden b/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden index 6994537d4f4e..0445877995cb 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden @@ -16,6 +16,7 @@ { "name": "envoy.http_connection_manager", "config": { + "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/api/agent.go b/api/agent.go index 1ef331247fd9..7a5c4272096a 100644 --- a/api/agent.go +++ b/api/agent.go @@ -103,6 +103,7 @@ type AgentServiceConnectProxyConfig struct { Config map[string]interface{} `json:",omitempty" bexpr:"-"` Upstreams []Upstream `json:",omitempty"` MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` } // AgentMember represents a cluster member known to the agent diff --git a/api/config_entry.go b/api/config_entry.go index 1588f2eed8ef..f91245454b27 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -56,6 +56,41 @@ type MeshGatewayConfig struct { Mode MeshGatewayMode `json:",omitempty"` } +// ExposeConfig describes HTTP paths to expose through Envoy outside of Connect. +// Users can expose individual paths and/or all HTTP/GRPC paths for checks. +type ExposeConfig struct { + // Checks defines whether paths associated with Consul checks will be exposed. + // This flag triggers exposing all HTTP and GRPC check paths registered for the service. + Checks bool `json:",omitempty"` + + // Paths is the list of paths exposed through the proxy. + Paths []Path `json:",omitempty"` +} + +type Path struct { + // ListenerPort defines the port of the proxy's listener for exposed paths. + ListenerPort int `json:",omitempty"` + + // Path is the path to expose through the proxy, ie. "/metrics." + Path string `json:",omitempty"` + + // LocalPathPort is the port that the service is listening on for the given path. + LocalPathPort int `json:",omitempty"` + + // Protocol describes the upstream's service protocol. + // Valid values are "http" and "http2", defaults to "http" + Protocol string `json:",omitempty"` + + // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. + TLSSkipVerify bool `json:",omitempty"` + + // CAFile is the path to the PEM encoded CA cert used to verify client certificates. + CAFile string `json:",omitempty"` + + // CACert contains the PEM encoded CA file read from CAFile + CACert string `json:",omitempty"` +} + type ServiceConfigEntry struct { Kind string Name string From ba19c389b4e515c09ccd56880c2a2d42333bb0fe Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 16:44:53 -0600 Subject: [PATCH 39/80] Move expose config validation to NodeService func --- agent/agent_endpoint.go | 2 +- agent/config/builder.go | 3 - agent/structs/connect_proxy_config.go | 30 +--- agent/structs/connect_proxy_config_test.go | 158 --------------------- agent/structs/structs.go | 30 ++++ agent/structs/structs_test.go | 50 +++++++ agent/structs/testing_catalog.go | 26 ++++ agent/xds/clusters.go | 1 + agent/xds/listeners.go | 1 + 9 files changed, 116 insertions(+), 185 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 78a8df6a69d2..e1581b84a2f6 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -24,7 +24,7 @@ import ( "github.com/hashicorp/consul/lib/file" "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/types" - "github.com/hashicorp/go-bexpr" + bexpr "github.com/hashicorp/go-bexpr" "github.com/hashicorp/logutils" "github.com/hashicorp/serf/coordinate" "github.com/hashicorp/serf/serf" diff --git a/agent/config/builder.go b/agent/config/builder.go index 731f3a72a335..4aaa474d0899 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1362,9 +1362,6 @@ func (b *Builder) exposeConfVal(v *ExposeConfig) structs.ExposeConfig { out.Checks = b.boolVal(v.Checks) out.Paths = b.pathsVal(v.Paths) - if err := out.Finalize(); err != nil { - b.err = multierror.Append(b.err, err) - } return out } diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index db9260070c10..05d11fca2c80 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -3,11 +3,10 @@ package structs import ( "encoding/json" "fmt" - "io/ioutil" - "strings" - "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" + "io/ioutil" + "log" ) const ( @@ -360,36 +359,21 @@ type Path struct { } // Finalize validates ExposeConfig and sets default values -func (e *ExposeConfig) Finalize() error { - var known = make(map[string]bool) +func (e *ExposeConfig) Finalize(l *log.Logger) { for i := 0; i < len(e.Paths); i++ { path := &e.Paths[i] - if seen := known[path.Path]; seen { - return fmt.Errorf("duplicate paths exposed") - } - known[path.Path] = true - - if path.ListenerPort <= 0 || path.ListenerPort > 65535 { - return fmt.Errorf("invalid listener port: %d", path.ListenerPort) + if path.Protocol == "" { + path.Protocol = defaultExposeProtocol } if path.CAFile != "" { b, err := ioutil.ReadFile(path.CAFile) if err != nil { - return fmt.Errorf("failed to read CAFile '%s': %v", path.CAFile, err) + l.Printf("[WARN] envoy: failed to read CAFile '%s': %v", path.CAFile, err) + continue } path.CACert = string(b) } - - path.Protocol = strings.ToLower(path.Protocol) - if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { - return fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", - path.Protocol, path.Path) - } - if path.Protocol == "" { - path.Protocol = defaultExposeProtocol - } } - return nil } diff --git a/agent/structs/connect_proxy_config_test.go b/agent/structs/connect_proxy_config_test.go index 6326ab253c33..bc1dd0f50d17 100644 --- a/agent/structs/connect_proxy_config_test.go +++ b/agent/structs/connect_proxy_config_test.go @@ -3,9 +3,6 @@ package structs import ( "encoding/json" "fmt" - "github.com/stretchr/testify/assert" - "io/ioutil" - "os" "testing" "github.com/hashicorp/consul/api" @@ -273,158 +270,3 @@ func TestValidateMeshGatewayMode(t *testing.T) { }) } } - -func TestExposeConfig_Finalize(t *testing.T) { - t.Parallel() - - tmpDir, err := ioutil.TempDir("", "exposeconfig_") - if err != nil { - t.Fatalf("failed to create tempdir: %v", err) - } - defer os.RemoveAll(tmpDir) - - tmpFile, err := ioutil.TempFile(tmpDir, "CAFile") - if err != nil { - t.Fatalf("failed to create tempfile: %v", err) - } - defer tmpFile.Close() - - type fields struct { - Checks bool - Paths []Path - } - tests := []struct { - name string - fields fields - want ExposeConfig - wantErr bool - errMsg string - }{ - { - name: "duplicate path", - fields: fields{ - Paths: []Path{ - { - LocalPathPort: 80, - ListenerPort: 80, - Path: "/metrics", - }, - { - LocalPathPort: 80, - ListenerPort: 80, - Path: "/metrics", - }, - }, - }, - wantErr: true, - errMsg: "duplicate paths exposed", - }, - { - name: "negative listener port", - fields: fields{ - Paths: []Path{ - { - ListenerPort: -1, - }, - }, - }, - wantErr: true, - errMsg: "invalid listener port: -1", - }, - { - name: "listener port too large", - fields: fields{ - Paths: []Path{ - { - ListenerPort: 65536, - }, - }, - }, - wantErr: true, - errMsg: "invalid listener port: 65536", - }, - { - name: "protocol not supported", - fields: fields{ - Paths: []Path{ - { - Path: "/metrics", - LocalPathPort: 80, - ListenerPort: 80, - Protocol: "tcp", - }, - }, - }, - wantErr: true, - errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", - }, - { - name: "protocol not supported", - fields: fields{ - Paths: []Path{ - { - Path: "/metrics", - LocalPathPort: 80, - ListenerPort: 80, - Protocol: "tcp", - }, - }, - }, - wantErr: true, - errMsg: "protocol 'tcp' not supported for path: /metrics, must be http or http2", - }, - { - name: "default to http when no protocol", - fields: fields{ - Paths: []Path{ - { - LocalPathPort: 80, - ListenerPort: 80, - }, - }, - }, - wantErr: false, - want: ExposeConfig{ - Paths: []Path{ - {LocalPathPort: 80, ListenerPort: 80, Protocol: "http"}, - }, - }, - }, - { - name: "lowercase protocol", - fields: fields{ - Paths: []Path{ - { - LocalPathPort: 80, - ListenerPort: 80, - Protocol: "HTTP2", - }, - }, - }, - wantErr: false, - want: ExposeConfig{ - Paths: []Path{ - {LocalPathPort: 80, ListenerPort: 80, Protocol: "http2"}, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - e := &ExposeConfig{ - Checks: tt.fields.Checks, - Paths: tt.fields.Paths, - } - err := e.Finalize() - if (err != nil) != tt.wantErr { - t.Errorf("Finalize() got error = %v, wantErr %v", err, tt.wantErr) - } - if tt.wantErr && tt.errMsg != err.Error() { - t.Errorf("Finalize() got error: '%v', want: '%s'", err, tt.errMsg) - } - if !tt.wantErr { - assert.Equal(t, &tt.want, e) - } - }) - } -} diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 7880fef0b486..145b53fe69bc 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "net" + "os" "reflect" "regexp" "sort" @@ -946,6 +947,35 @@ func (s *NodeService) Validate() error { } bindAddrs[addr] = struct{}{} } + var known = make(map[string]bool) + for _, path := range s.Proxy.Expose.Paths { + if path.Path == "" { + result = multierror.Append(result, fmt.Errorf("empty path exposed")) + } + + if seen := known[path.Path]; seen { + result = multierror.Append(result, fmt.Errorf("duplicate paths exposed")) + } + known[path.Path] = true + + if path.ListenerPort <= 0 || path.ListenerPort > 65535 { + result = multierror.Append(result, fmt.Errorf("invalid listener port: %d", path.ListenerPort)) + } + + if path.CAFile != "" { + _, err := os.Stat(path.CAFile) + if err != nil { + result = multierror.Append(result, fmt.Errorf("failed to find CAFile '%s': %v", path.CAFile, err)) + } + } + + path.Protocol = strings.ToLower(path.Protocol) + if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { + result = multierror.Append(result, + fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", + path.Protocol, path.Path)) + } + } } // MeshGateway validation diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index 1549ef548bbe..333cdcd7ff19 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -414,6 +414,56 @@ func TestStructs_NodeService_ValidateMeshGateway(t *testing.T) { } } +func TestStructs_NodeService_ValidateExposeConfig(t *testing.T) { + type testCase struct { + Modify func(*NodeService) + Err string + } + cases := map[string]testCase{ + "valid": { + func(x *NodeService) {}, + "", + }, + "empty path": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].Path = "" }, + "empty path exposed", + }, + "invalid port negative": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].ListenerPort = -1 }, + "invalid listener port", + }, + "invalid port too large": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].ListenerPort = 65536 }, + "invalid listener port", + }, + "duplicate paths": { + func(x *NodeService) { + x.Proxy.Expose.Paths[0].Path = "/metrics" + x.Proxy.Expose.Paths[1].Path = "/metrics" + }, + "duplicate paths exposed", + }, + "protocol not supported": { + func(x *NodeService) { x.Proxy.Expose.Paths[0].Protocol = "foo" }, + "protocol 'foo' not supported for path", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ns := TestNodeServiceExpose(t) + tc.Modify(ns) + + err := ns.Validate() + if tc.Err == "" { + require.NoError(t, err) + } else { + require.Contains(t, strings.ToLower(err.Error()), strings.ToLower(tc.Err)) + } + }) + } +} + func TestStructs_NodeService_ValidateConnectProxy(t *testing.T) { cases := []struct { Name string diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index e1f847bf6f5a..26ba4a6c6100 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -50,6 +50,32 @@ func TestNodeServiceProxy(t testing.T) *NodeService { } } +func TestNodeServiceExpose(t testing.T) *NodeService { + return &NodeService{ + Kind: ServiceKindConnectProxy, + Service: "test-svc", + Address: "localhost", + Port: 8080, + Proxy: ConnectProxyConfig{ + DestinationServiceName: "web", + Expose: ExposeConfig{ + Paths: []Path{ + { + Path: "/foo", + LocalPathPort: 80, + ListenerPort: 80, + }, + { + Path: "/bar", + LocalPathPort: 80, + ListenerPort: 80, + }, + }, + }, + }, + } +} + // TestNodeServiceMeshGateway returns a *NodeService representing a valid Mesh Gateway func TestNodeServiceMeshGateway(t testing.T) *NodeService { return TestNodeServiceMeshGatewayWithAddrs(t, diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 70b2fa87a5fb..bb2e93aa5a47 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -74,6 +74,7 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh } } + cfgSnap.Proxy.Expose.Finalize(s.Logger) paths := cfgSnap.Proxy.Expose.Paths // Add service health checks to the list of paths to create clusters for if needed diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 28269b5216c0..62a54b78039f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -74,6 +74,7 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps resources[i+1] = upstreamListener } + cfgSnap.Proxy.Expose.Finalize(s.Logger) paths := cfgSnap.Proxy.Expose.Paths // Add service health checks to the list of paths to create listeners for if needed From 404b323833b374e8a8278133b2c3a25c5512375c Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 16:50:21 -0600 Subject: [PATCH 40/80] Avoid panic in expose.checks check --- agent/agent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 2171feda99e1..90bbd426f543 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2432,7 +2432,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, TLSClientConfig: tlsClientConfig, } - if service.Proxy.Expose.Checks { + if service != nil && service.Proxy.Expose.Checks { port, err := a.listenerPort(service.ID, string(http.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) @@ -2495,7 +2495,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, TLSClientConfig: tlsClientConfig, } - if service.Proxy.Expose.Checks { + if service != nil && service.Proxy.Expose.Checks { port, err := a.listenerPort(service.ID, string(grpc.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) From 965144172f33c3c3808a9c88d73ea085d802f7c9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 18:08:27 -0600 Subject: [PATCH 41/80] Fix broken agent/api tests --- agent/agent.go | 4 +++- agent/agent_endpoint_test.go | 4 ++-- agent/structs/connect_proxy_config.go | 28 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 90bbd426f543..6fc6ad7eefad 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2244,7 +2244,9 @@ func (a *Agent) removeServiceLocked(serviceID string, persist bool) error { // Reset the HTTP check targets if they were exposed through a proxy // If this is not a proxy or checks were not exposed then this is a no-op svc := a.State.Service(serviceID) - a.resetExposedChecks(svc.Proxy.DestinationServiceID) + if svc != nil { + a.resetExposedChecks(svc.Proxy.DestinationServiceID) + } checks := a.State.Checks() var checkIDs []types.CheckID diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index 500994ffe42b..afbc36c69058 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -325,7 +325,7 @@ func TestAgent_Service(t *testing.T) { Service: "web-sidecar-proxy", Port: 8000, Proxy: expectProxy.ToAPI(), - ContentHash: "f5826efc5ffc207a", + ContentHash: "4c7d5f8d3748be6d", Weights: api.AgentWeights{ Passing: 1, Warning: 1, @@ -337,7 +337,7 @@ func TestAgent_Service(t *testing.T) { // Copy and modify updatedResponse := *expectedResponse updatedResponse.Port = 9999 - updatedResponse.ContentHash = "c8cb04cb77ef33d8" + updatedResponse.ContentHash = "713435ba1f5badcf" // Simple response for non-proxy service registered in TestAgent config expectWebResponse := &api.AgentService{ diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 05d11fca2c80..00715b57e6c5 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -147,6 +147,7 @@ func (c *ConnectProxyConfig) ToAPI() *api.AgentServiceConnectProxyConfig { Config: c.Config, Upstreams: c.Upstreams.ToAPI(), MeshGateway: c.MeshGateway.ToAPI(), + Expose: c.Expose.ToAPI(), } } @@ -358,6 +359,33 @@ type Path struct { CACert string } +func (e *ExposeConfig) ToAPI() api.ExposeConfig { + paths := make([]api.Path, 0) + for _, p := range e.Paths { + paths = append(paths, p.ToAPI()) + } + if e.Paths == nil { + paths = nil + } + + return api.ExposeConfig{ + Checks: e.Checks, + Paths: paths, + } +} + +func (p *Path) ToAPI() api.Path { + return api.Path{ + ListenerPort: p.ListenerPort, + Path: p.Path, + LocalPathPort: p.LocalPathPort, + Protocol: p.Protocol, + TLSSkipVerify: p.TLSSkipVerify, + CAFile: p.CAFile, + CACert: p.CACert, + } +} + // Finalize validates ExposeConfig and sets default values func (e *ExposeConfig) Finalize(l *log.Logger) { for i := 0; i < len(e.Paths); i++ { From f2b8fe8e4dc9563f5358ae1383b63aab2c73dc48 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 18:26:30 -0600 Subject: [PATCH 42/80] Rename Path to ExposePath --- agent/config/builder.go | 6 +++--- agent/config/config.go | 4 ++-- agent/config/runtime_test.go | 6 +++--- agent/structs/connect_proxy_config.go | 12 ++++++------ agent/structs/testing_catalog.go | 2 +- agent/xds/listeners.go | 4 ++-- api/config_entry.go | 4 ++-- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/agent/config/builder.go b/agent/config/builder.go index 4aaa474d0899..cd2c3261dcbb 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1365,10 +1365,10 @@ func (b *Builder) exposeConfVal(v *ExposeConfig) structs.ExposeConfig { return out } -func (b *Builder) pathsVal(v []Path) []structs.Path { - paths := make([]structs.Path, len(v)) +func (b *Builder) pathsVal(v []ExposePath) []structs.ExposePath { + paths := make([]structs.ExposePath, len(v)) for i, p := range v { - paths[i] = structs.Path{ + paths[i] = structs.ExposePath{ ListenerPort: b.intVal(p.ListenerPort), Path: b.stringVal(p.Path), LocalPathPort: b.intVal(p.LocalPathPort), diff --git a/agent/config/config.go b/agent/config/config.go index 4f4d53a71db0..db957106f0e0 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -533,10 +533,10 @@ type ExposeConfig struct { Port *int `json:"port,omitempty" hcl:"port" mapstructure:"port"` // Paths is the list of paths exposed through the proxy. - Paths []Path `json:"paths,omitempty" hcl:"paths" mapstructure:"paths"` + Paths []ExposePath `json:"paths,omitempty" hcl:"paths" mapstructure:"paths"` } -type Path struct { +type ExposePath struct { // ListenerPort defines the port of the proxy's listener for exposed paths. ListenerPort *int `json:"listener_port,omitempty" hcl:"listener_port" mapstructure:"listener_port"` diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 60b6a1a62d30..a6109d8c2a9d 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -2445,7 +2445,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { Proxy: &structs.ConnectProxyConfig{ Expose: structs.ExposeConfig{ Checks: true, - Paths: []structs.Path{ + Paths: []structs.ExposePath{ { Path: "/health", LocalPathPort: 8080, @@ -2574,7 +2574,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { Proxy: &structs.ConnectProxyConfig{ Expose: structs.ExposeConfig{ Checks: true, - Paths: []structs.Path{ + Paths: []structs.ExposePath{ { Path: "/health", LocalPathPort: 8080, @@ -5170,7 +5170,7 @@ func TestFullConfig(t *testing.T) { }, Expose: structs.ExposeConfig{ Checks: true, - Paths: []structs.Path{ + Paths: []structs.ExposePath{ { Path: "/health", LocalPathPort: 8080, diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 00715b57e6c5..50506451a85f 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -332,14 +332,14 @@ type ExposeConfig struct { Checks bool `mapstructure:"checks"` // Paths is the list of paths exposed through the proxy. - Paths []Path `mapstructure:"paths"` + Paths []ExposePath `mapstructure:"paths"` } -type Path struct { +type ExposePath struct { // ListenerPort defines the port of the proxy's listener for exposed paths. ListenerPort int `mapstructure:"listener_port"` - // Path is the path to expose through the proxy, ie. "/metrics." + // ExposePath is the path to expose through the proxy, ie. "/metrics." Path string `mapstructure:"path"` // LocalPathPort is the port that the service is listening on for the given path. @@ -360,7 +360,7 @@ type Path struct { } func (e *ExposeConfig) ToAPI() api.ExposeConfig { - paths := make([]api.Path, 0) + paths := make([]api.ExposePath, 0) for _, p := range e.Paths { paths = append(paths, p.ToAPI()) } @@ -374,8 +374,8 @@ func (e *ExposeConfig) ToAPI() api.ExposeConfig { } } -func (p *Path) ToAPI() api.Path { - return api.Path{ +func (p *ExposePath) ToAPI() api.ExposePath { + return api.ExposePath{ ListenerPort: p.ListenerPort, Path: p.Path, LocalPathPort: p.LocalPathPort, diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index 26ba4a6c6100..ddc5fb4efbf3 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -59,7 +59,7 @@ func TestNodeServiceExpose(t testing.T) *NodeService { Proxy: ConnectProxyConfig{ DestinationServiceName: "web", Expose: ExposeConfig{ - Paths: []Path{ + Paths: []ExposePath{ { Path: "/foo", LocalPathPort: 80, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 62a54b78039f..ef006259634f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -106,11 +106,11 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps return resources, nil } -func parseCheckPath(check structs.CheckType) (structs.Path, error) { +func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { grpcRE := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") httpRE := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) - var path structs.Path + var path structs.ExposePath var err error if check.HTTP != "" { diff --git a/api/config_entry.go b/api/config_entry.go index f91245454b27..5f77dbac1cb0 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -64,10 +64,10 @@ type ExposeConfig struct { Checks bool `json:",omitempty"` // Paths is the list of paths exposed through the proxy. - Paths []Path `json:",omitempty"` + Paths []ExposePath `json:",omitempty"` } -type Path struct { +type ExposePath struct { // ListenerPort defines the port of the proxy's listener for exposed paths. ListenerPort int `json:",omitempty"` From c4c178060a4bec8385f399608b224b1cbac492d1 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 20:20:40 -0600 Subject: [PATCH 43/80] Add tests for check proxy addr setting/resetting --- agent/agent.go | 19 ++- agent/agent_test.go | 235 ++++++++++++++++++++++++++++++++++++- agent/checks/check.go | 2 +- agent/checks/check_test.go | 65 +++++++++- agent/checks/grpc_test.go | 57 +++++++++ 5 files changed, 371 insertions(+), 7 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index d190933ed2f1..a8d6cff7e2e3 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2382,6 +2382,17 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, if chkType.OutputMaxSize > 0 && maxOutputSize > chkType.OutputMaxSize { maxOutputSize = chkType.OutputMaxSize } + + // Get the address of the proxy for this service if it exists + // Need its config to know whether we should reroute checks to it + var proxy *structs.NodeService + services := a.State.Services() + for _, svc := range services { + if svc.Proxy.DestinationServiceID == service.ID { + proxy = svc + } + } + switch { case chkType.IsTTL(): @@ -2435,13 +2446,13 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, TLSClientConfig: tlsClientConfig, } - if service != nil && service.Proxy.Expose.Checks { + if proxy != nil && proxy.Proxy.Expose.Checks { port, err := a.listenerPort(service.ID, string(http.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) return err } - addr := httpInjectAddr(http.HTTP, service.Proxy.LocalServiceAddress, port) + addr := httpInjectAddr(http.HTTP, proxy.Proxy.LocalServiceAddress, port) http.ProxyHTTP = addr } @@ -2498,13 +2509,13 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, TLSClientConfig: tlsClientConfig, } - if service != nil && service.Proxy.Expose.Checks { + if proxy != nil && proxy.Proxy.Expose.Checks { port, err := a.listenerPort(service.ID, string(grpc.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) return err } - addr := grpcInjectAddr(grpc.GRPC, service.Proxy.LocalServiceAddress, port) + addr := grpcInjectAddr(grpc.GRPC, proxy.Proxy.LocalServiceAddress, port) grpc.ProxyGRPC = addr } diff --git a/agent/agent_test.go b/agent/agent_test.go index 992341c8b2d4..000f0e304094 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -29,7 +29,7 @@ import ( "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/types" - uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/go-uuid" "github.com/pascaldekloe/goe/verify" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -3546,3 +3546,236 @@ func TestAgent_httpInjectAddr(t *testing.T) { }) } } + +func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + + // Register a service without a ProxyAddr + svc := &structs.NodeService{ + ID: "web", + Service: "web", + Address: "localhost", + Port: 8080, + } + chks := []*structs.CheckType{ + { + CheckID: "http", + HTTP: "http://localhost:8080/mypath?query", + Interval: 20 * time.Millisecond, + TLSSkipVerify: true, + }, + { + CheckID: "grpc", + GRPC: "localhost:8080/myservice", + Interval: 20 * time.Millisecond, + TLSSkipVerify: true, + }, + } + if err := a.AddService(svc, chks, false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add svc: %v", err) + } + + // Register a proxy and expose HTTP checks + // This should trigger setting ProxyHTTP and ProxyGRPC in the checks + proxy := &structs.NodeService{ + Kind: "connect-proxy", + ID: "web-proxy", + Service: "web-proxy", + Address: "localhost", + Port: 21500, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "web", + DestinationServiceID: "web", + LocalServiceAddress: "localhost", + LocalServicePort: 8080, + MeshGateway: structs.MeshGatewayConfig{}, + Expose: structs.ExposeConfig{ + Checks: true, + }, + }, + } + if err := a.AddService(proxy, nil, false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add svc: %v", err) + } + + retry.Run(t, func(r *retry.R) { + chks := a.ServiceHTTPChecks("web") + + got := chks[0].ProxyHTTP + if got == "" { + r.Fatal("proxyHTTP addr not set in check") + } + + want := "http://localhost:21500/mypath?query" + if got != want { + r.Fatalf("unexpected proxy addr in check, want: %s, got: %s", want, got) + } + }) + + retry.Run(t, func(r *retry.R) { + chks := a.ServiceHTTPChecks("web") + + // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPChecks + got := chks[1].ProxyGRPC + if got == "" { + r.Fatal("ProxyGRPC addr not set in check") + } + + want := "localhost:21501/myservice" + if got != want { + r.Fatalf("unexpected proxy addr in check, want: %s, got: %s", want, got) + } + }) + + // Re-register a proxy and disable exposing HTTP checks + // This should trigger resetting ProxyHTTP and ProxyGRPC to empty strings + proxy = &structs.NodeService{ + Kind: "connect-proxy", + ID: "web-proxy", + Service: "web-proxy", + Address: "localhost", + Port: 21500, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "web", + DestinationServiceID: "web", + LocalServiceAddress: "localhost", + LocalServicePort: 8080, + MeshGateway: structs.MeshGatewayConfig{}, + Expose: structs.ExposeConfig{ + Checks: false, + }, + }, + } + if err := a.AddService(proxy, nil, false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add svc: %v", err) + } + + retry.Run(t, func(r *retry.R) { + chks := a.ServiceHTTPChecks("web") + + got := chks[0].ProxyHTTP + if got != "" { + r.Fatal("ProxyHTTP addr was not reset") + } + }) + + retry.Run(t, func(r *retry.R) { + chks := a.ServiceHTTPChecks("web") + + // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPChecks + got := chks[1].ProxyGRPC + if got != "" { + r.Fatal("ProxyGRPC addr was not reset") + } + }) +} + +func TestAgent_RerouteNewHTTPChecks(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + + // Register a service without a ProxyAddr + svc := &structs.NodeService{ + ID: "web", + Service: "web", + Address: "localhost", + Port: 8080, + } + if err := a.AddService(svc, nil, false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add svc: %v", err) + } + + // Register a proxy and expose HTTP checks + proxy := &structs.NodeService{ + Kind: "connect-proxy", + ID: "web-proxy", + Service: "web-proxy", + Address: "localhost", + Port: 21500, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "web", + DestinationServiceID: "web", + LocalServiceAddress: "localhost", + LocalServicePort: 8080, + MeshGateway: structs.MeshGatewayConfig{}, + Expose: structs.ExposeConfig{ + Checks: true, + }, + }, + } + if err := a.AddService(proxy, nil, false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add svc: %v", err) + } + + checks := []*structs.HealthCheck{ + { + CheckID: "http", + Name: "http", + ServiceID: "web", + Status: api.HealthCritical, + }, + { + CheckID: "grpc", + Name: "grpc", + ServiceID: "web", + Status: api.HealthCritical, + }, + } + chkTypes := []*structs.CheckType{ + { + CheckID: "http", + HTTP: "http://localhost:8080/mypath?query", + Interval: 20 * time.Millisecond, + TLSSkipVerify: true, + }, + { + CheckID: "grpc", + GRPC: "localhost:8080/myservice", + Interval: 20 * time.Millisecond, + TLSSkipVerify: true, + }, + } + + // ProxyGRPC and ProxyHTTP should be set when creating check + // since proxy.expose.checks is enabled on the proxy + if err := a.AddCheck(checks[0], chkTypes[0], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add check: %v", err) + } + if err := a.AddCheck(checks[1], chkTypes[1], false, "", ConfigSourceLocal); err != nil { + t.Fatalf("failed to add check: %v", err) + } + + retry.Run(t, func(r *retry.R) { + chks := a.ServiceHTTPChecks("web") + + got := chks[0].ProxyHTTP + if got == "" { + r.Fatal("ProxyHTTP addr not set in check") + } + + want := "http://localhost:21500/mypath?query" + if got != want { + r.Fatalf("unexpected proxy addr in http check, want: %s, got: %s", want, got) + } + }) + + retry.Run(t, func(r *retry.R) { + chks := a.ServiceHTTPChecks("web") + + // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPChecks + got := chks[1].ProxyGRPC + if got == "" { + r.Fatal("ProxyGRPC addr not set in check") + } + + want := "localhost:21501/myservice" + if got != want { + r.Fatalf("unexpected proxy addr in grpc check, want: %s, got: %s", want, got) + } + }) +} diff --git a/agent/checks/check.go b/agent/checks/check.go index bb4ca538e99a..0fbcd4a85cdf 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -413,7 +413,7 @@ func (c *CheckHTTP) check() { target := c.HTTP if c.ProxyHTTP != "" { - target = c.HTTP + target = c.ProxyHTTP } req, err := http.NewRequest(method, target, nil) diff --git a/agent/checks/check_test.go b/agent/checks/check_test.go index 4cb4edf66920..7cbcf2748122 100644 --- a/agent/checks/check_test.go +++ b/agent/checks/check_test.go @@ -19,7 +19,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/types" - uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/go-uuid" ) func uniqueID() string { @@ -328,6 +328,69 @@ func TestCheckHTTP(t *testing.T) { } } +func TestCheckHTTP_Proxied(t *testing.T) { + t.Parallel() + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Proxy Server") + })) + defer proxy.Close() + + notif := mock.NewNotify() + check := &CheckHTTP{ + Notify: notif, + CheckID: types.CheckID("foo"), + HTTP: "", + Method: "GET", + OutputMaxSize: DefaultBufSize, + Interval: 10 * time.Millisecond, + Logger: log.New(ioutil.Discard, uniqueID(), log.LstdFlags), + ProxyHTTP: proxy.URL, + } + + check.Start() + defer check.Stop() + + // If ProxyHTTP is set, check() reqs should go to that address + retry.Run(t, func(r *retry.R) { + output := notif.Output("foo") + if !strings.Contains(output, "Proxy Server") { + r.Fatalf("c.ProxyHTTP server did not receive request, but should") + } + }) +} + +func TestCheckHTTP_NotProxied(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Original Server") + })) + defer server.Close() + + notif := mock.NewNotify() + check := &CheckHTTP{ + Notify: notif, + CheckID: types.CheckID("foo"), + HTTP: server.URL, + Method: "GET", + OutputMaxSize: DefaultBufSize, + Interval: 10 * time.Millisecond, + Logger: log.New(ioutil.Discard, uniqueID(), log.LstdFlags), + ProxyHTTP: "", + } + check.Start() + defer check.Stop() + + // If ProxyHTTP is not set, check() reqs should go to the address in CheckHTTP.HTTP + retry.Run(t, func(r *retry.R) { + output := notif.Output("foo") + if !strings.Contains(output, "Original Server") { + r.Fatalf("server did not receive request") + } + }) +} + func TestCheckHTTPTCP_BigTimeout(t *testing.T) { testCases := []struct { timeoutIn, intervalIn, timeoutWant time.Duration diff --git a/agent/checks/grpc_test.go b/agent/checks/grpc_test.go index ebfe8aa518dc..ad869cb6d2b7 100644 --- a/agent/checks/grpc_test.go +++ b/agent/checks/grpc_test.go @@ -4,6 +4,11 @@ import ( "crypto/tls" "flag" "fmt" + "github.com/hashicorp/consul/agent/mock" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/hashicorp/consul/types" + "io/ioutil" "log" "net" "os" @@ -96,3 +101,55 @@ func TestCheck(t *testing.T) { }) } } + +func TestGRPC_Proxied(t *testing.T) { + t.Parallel() + + notif := mock.NewNotify() + check := &CheckGRPC{ + Notify: notif, + CheckID: types.CheckID("foo"), + GRPC: "", + Interval: 10 * time.Millisecond, + Logger: log.New(ioutil.Discard, uniqueID(), log.LstdFlags), + ProxyGRPC: server, + } + check.Start() + defer check.Stop() + + // If ProxyGRPC is set, check() reqs should go to that address + retry.Run(t, func(r *retry.R) { + if got, want := notif.Updates("foo"), 2; got < want { + r.Fatalf("got %d updates want at least %d", got, want) + } + if got, want := notif.State("foo"), api.HealthPassing; got != want { + r.Fatalf("got state %q want %q", got, want) + } + }) +} + +func TestGRPC_NotProxied(t *testing.T) { + t.Parallel() + + notif := mock.NewNotify() + check := &CheckGRPC{ + Notify: notif, + CheckID: types.CheckID("foo"), + GRPC: server, + Interval: 10 * time.Millisecond, + Logger: log.New(ioutil.Discard, uniqueID(), log.LstdFlags), + ProxyGRPC: "", + } + check.Start() + defer check.Stop() + + // If ProxyGRPC is not set, check() reqs should go to check.GRPC + retry.Run(t, func(r *retry.R) { + if got, want := notif.Updates("foo"), 2; got < want { + r.Fatalf("got %d updates want at least %d", got, want) + } + if got, want := notif.State("foo"), api.HealthPassing; got != want { + r.Fatalf("got state %q want %q", got, want) + } + }) +} From 5be00c75263f353b9c9b0cebf770cf4545446a69 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 23:16:38 -0600 Subject: [PATCH 44/80] Add more tests and config entry support --- agent/agent_endpoint.go | 2 +- agent/agent_endpoint_test.go | 40 ++++++++++++++++++ agent/agent_test.go | 4 ++ agent/config_endpoint_test.go | 58 +++++++++++++++++++++++++++ agent/consul/config_endpoint.go | 7 ++++ agent/consul/config_endpoint_test.go | 43 ++++++++++++++++++++ agent/structs/config_entry.go | 3 ++ agent/structs/connect_proxy_config.go | 16 ++++---- api/agent_test.go | 42 +++++++++++++++++++ api/config_entry.go | 2 + 10 files changed, 208 insertions(+), 9 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index e1581b84a2f6..78a8df6a69d2 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -24,7 +24,7 @@ import ( "github.com/hashicorp/consul/lib/file" "github.com/hashicorp/consul/logger" "github.com/hashicorp/consul/types" - bexpr "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/go-bexpr" "github.com/hashicorp/logutils" "github.com/hashicorp/serf/coordinate" "github.com/hashicorp/serf/serf" diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index afbc36c69058..7136b45bd782 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -5251,3 +5251,43 @@ func TestAgent_HostBadACL(t *testing.T) { assert.Equal(http.StatusOK, resp.Code) assert.Nil(respRaw) } + +// Thie tests that a proxy with an ExposeConfig is returned as expected. +func TestAgent_Services_ExposeConfig(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + srv1 := &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "proxy-id", + Service: "proxy-name", + Port: 8443, + Proxy: structs.ConnectProxyConfig{ + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.ExposePath{ + { + ListenerPort: 8080, + LocalPathPort: 21500, + Protocol: "http2", + Path: "/metrics", + }, + }, + }, + }, + } + a.State.AddService(srv1, "") + + req, _ := http.NewRequest("GET", "/v1/agent/services", nil) + obj, err := a.srv.AgentServices(nil, req) + require.NoError(t, err) + val := obj.(map[string]*api.AgentService) + require.Len(t, val, 1) + actual := val["proxy-id"] + require.NotNil(t, actual) + require.Equal(t, api.ServiceKindConnectProxy, actual.Kind) + require.Equal(t, srv1.Proxy.ToAPI(), actual.Proxy) +} diff --git a/agent/agent_test.go b/agent/agent_test.go index 000f0e304094..7e9d7595955c 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3553,6 +3553,8 @@ func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { a := NewTestAgent(t, t.Name(), "") defer a.Shutdown() + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + // Register a service without a ProxyAddr svc := &structs.NodeService{ ID: "web", @@ -3679,6 +3681,8 @@ func TestAgent_RerouteNewHTTPChecks(t *testing.T) { a := NewTestAgent(t, t.Name(), "") defer a.Shutdown() + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + // Register a service without a ProxyAddr svc := &structs.NodeService{ ID: "web", diff --git a/agent/config_endpoint_test.go b/agent/config_endpoint_test.go index 50f2ce6a56dc..1fcf58b4efd3 100644 --- a/agent/config_endpoint_test.go +++ b/agent/config_endpoint_test.go @@ -373,3 +373,61 @@ func TestConfig_Apply_Decoding(t *testing.T) { } }) } + +func TestConfig_Apply_ProxyDefaultsExpose(t *testing.T) { + t.Parallel() + + a := NewTestAgent(t, t.Name(), "") + defer a.Shutdown() + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + + // Create some config entries. + body := bytes.NewBuffer([]byte(` + { + "Kind": "proxy-defaults", + "Name": "global", + "Expose": { + "Checks": true, + "Paths": [ + { + "LocalPathPort": 8080, + "ListenerPort": 21500, + "Path": "/healthz", + "Protocol": "http2" + } + ] + } + }`)) + + req, _ := http.NewRequest("PUT", "/v1/config", body) + resp := httptest.NewRecorder() + _, err := a.srv.ConfigApply(resp, req) + require.NoError(t, err) + require.Equal(t, 200, resp.Code, "!200 Response Code: %s", resp.Body.String()) + + // Get the remaining entry. + { + args := structs.ConfigEntryQuery{ + Kind: structs.ProxyDefaults, + Name: "global", + Datacenter: "dc1", + } + var out structs.ConfigEntryResponse + require.NoError(t, a.RPC("ConfigEntry.Get", &args, &out)) + require.NotNil(t, out.Entry) + entry := out.Entry.(*structs.ProxyConfigEntry) + + expose := structs.ExposeConfig{ + Checks: true, + Paths: []structs.ExposePath{ + { + LocalPathPort: 8080, + ListenerPort: 21500, + Path: "/healthz", + Protocol: "http2", + }, + }, + } + require.Equal(t, expose, entry.Expose) + } +} diff --git a/agent/consul/config_endpoint.go b/agent/consul/config_endpoint.go index 01834f8af09f..3edd82f9c0e0 100644 --- a/agent/consul/config_endpoint.go +++ b/agent/consul/config_endpoint.go @@ -275,11 +275,18 @@ func (c *ConfigEntry) ResolveServiceConfig(args *structs.ServiceConfigRequest, r } reply.ProxyConfig = mapCopy.(map[string]interface{}) reply.MeshGateway = proxyConf.MeshGateway + reply.Expose = proxyConf.Expose } reply.Index = index if serviceConf != nil { + if serviceConf.Expose.Checks { + reply.Expose.Checks = true + } + if len(serviceConf.Expose.Paths) >= 1 { + reply.Expose.Paths = serviceConf.Expose.Paths + } if serviceConf.MeshGateway.Mode != structs.MeshGatewayModeDefault { reply.MeshGateway.Mode = serviceConf.MeshGateway.Mode } diff --git a/agent/consul/config_endpoint_test.go b/agent/consul/config_endpoint_test.go index 9fdcd847be8a..32ff43a70132 100644 --- a/agent/consul/config_endpoint_test.go +++ b/agent/consul/config_endpoint_test.go @@ -1127,3 +1127,46 @@ operator = "write" require.NoError(msgpackrpc.CallWithCodec(codec, "ConfigEntry.ResolveServiceConfig", &args, &out)) } + +func TestConfigEntry_ProxyDefaultsExposeConfig(t *testing.T) { + t.Parallel() + + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + codec := rpcClient(t, s1) + defer codec.Close() + + expose := structs.ExposeConfig{ + Checks: true, + Paths: []structs.ExposePath{ + { + LocalPathPort: 8080, + ListenerPort: 21500, + Protocol: "http2", + Path: "/healthz", + }, + }, + } + + args := structs.ConfigEntryRequest{ + Datacenter: "dc1", + Entry: &structs.ProxyConfigEntry{ + Kind: "proxy-defaults", + Name: "global", + Expose: expose, + }, + } + + out := false + require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &args, &out)) + require.True(t, out) + + state := s1.fsm.State() + _, entry, err := state.ConfigEntry(nil, structs.ProxyDefaults, "global") + require.NoError(t, err) + + proxyConf, ok := entry.(*structs.ProxyConfigEntry) + require.True(t, ok) + require.Equal(t, expose, proxyConf.Expose) +} diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 13d16ff1bcff..839ae8ee6a3b 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -51,6 +51,7 @@ type ServiceConfigEntry struct { Name string Protocol string MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` ExternalSNI string `json:",omitempty"` @@ -116,6 +117,7 @@ type ProxyConfigEntry struct { Name string Config map[string]interface{} MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` RaftIndex } @@ -534,6 +536,7 @@ type ServiceConfigResponse struct { ProxyConfig map[string]interface{} UpstreamConfigs map[string]map[string]interface{} MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` QueryMeta } diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 50506451a85f..fab91d54d1ff 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -329,31 +329,31 @@ func UpstreamFromAPI(u api.Upstream) Upstream { type ExposeConfig struct { // Checks defines whether paths associated with Consul checks will be exposed. // This flag triggers exposing all HTTP and GRPC check paths registered for the service. - Checks bool `mapstructure:"checks"` + Checks bool `json:",omitempty"` // Paths is the list of paths exposed through the proxy. - Paths []ExposePath `mapstructure:"paths"` + Paths []ExposePath `json:",omitempty"` } type ExposePath struct { // ListenerPort defines the port of the proxy's listener for exposed paths. - ListenerPort int `mapstructure:"listener_port"` + ListenerPort int `json:",omitempty"` // ExposePath is the path to expose through the proxy, ie. "/metrics." - Path string `mapstructure:"path"` + Path string `json:",omitempty"` // LocalPathPort is the port that the service is listening on for the given path. - LocalPathPort int `mapstructure:"local_path_port"` + LocalPathPort int `json:",omitempty"` // Protocol describes the upstream's service protocol. // Valid values are "http" and "http2", defaults to "http" - Protocol string `mapstructure:"protocol"` + Protocol string `json:",omitempty"` // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. - TLSSkipVerify bool `mapstructure:"tls_skip_verify"` + TLSSkipVerify bool `json:",omitempty"` // CAFile is the path to the PEM encoded CA cert used to verify client certificates. - CAFile string `mapstructure:"ca_file"` + CAFile string `json:",omitempty"` // CACert contains the PEM encoded CA file read from CAFile CACert string diff --git a/api/agent_test.go b/api/agent_test.go index c98ad5d7b127..3b8e61f4a7b0 100644 --- a/api/agent_test.go +++ b/api/agent_test.go @@ -1567,3 +1567,45 @@ func TestAgentService_Register_MeshGateway(t *testing.T) { require.Contains(t, svc.Proxy.Config, "foo") require.Equal(t, "bar", svc.Proxy.Config["foo"]) } + +func TestAgentService_ExposeChecks(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + agent := c.Agent() + + path := ExposePath{ + LocalPathPort: 8080, + ListenerPort: 21500, + Path: "/metrics", + Protocol: "http2", + } + reg := AgentServiceRegistration{ + Kind: ServiceKindConnectProxy, + Name: "expose-proxy", + Address: "10.1.2.3", + Port: 8443, + Proxy: &AgentServiceConnectProxyConfig{ + DestinationServiceName: "expose", + Expose: ExposeConfig{ + Checks: true, + Paths: []ExposePath{ + path, + }, + }, + }, + } + + err := agent.ServiceRegister(®) + require.NoError(t, err) + + svc, _, err := agent.Service("expose-proxy", nil) + require.NoError(t, err) + require.NotNil(t, svc) + require.Equal(t, ServiceKindConnectProxy, svc.Kind) + require.NotNil(t, svc.Proxy) + require.Len(t, svc.Proxy.Expose.Paths, 1) + require.True(t, svc.Proxy.Expose.Checks) + require.Equal(t, path, svc.Proxy.Expose.Paths[0]) +} diff --git a/api/config_entry.go b/api/config_entry.go index 5f77dbac1cb0..59e9b5b41d14 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -96,6 +96,7 @@ type ServiceConfigEntry struct { Name string Protocol string `json:",omitempty"` MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` ExternalSNI string `json:",omitempty"` CreateIndex uint64 ModifyIndex uint64 @@ -122,6 +123,7 @@ type ProxyConfigEntry struct { Name string Config map[string]interface{} `json:",omitempty"` MeshGateway MeshGatewayConfig `json:",omitempty"` + Expose ExposeConfig `json:",omitempty"` CreateIndex uint64 ModifyIndex uint64 } From 35c80c01ddafc90102ebfde3dc8d85e463de7f62 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 4 Sep 2019 23:41:43 -0600 Subject: [PATCH 45/80] Add listener and cluster creation tests --- agent/proxycfg/testing.go | 29 ++++ agent/xds/clusters_test.go | 16 ++ agent/xds/listeners_test.go | 16 ++ .../expose-paths-local-app-paths.golden | 32 ++++ .../expose-paths-new-cluster-http2.golden | 60 +++++++ .../expose-paths-local-app-paths.golden | 154 +++++++++++++++++ .../expose-paths-new-cluster-http2.golden | 157 ++++++++++++++++++ 7 files changed, 464 insertions(+) create mode 100644 agent/xds/testdata/clusters/expose-paths-local-app-paths.golden create mode 100644 agent/xds/testdata/clusters/expose-paths-new-cluster-http2.golden create mode 100644 agent/xds/testdata/listeners/expose-paths-local-app-paths.golden create mode 100644 agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 3c484bf0cf19..7357f068dae2 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -1005,6 +1005,35 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot { } } +func TestConfigSnapshotExposeConfig(t testing.T) *ConfigSnapshot { + return &ConfigSnapshot{ + Kind: structs.ServiceKindConnectProxy, + Service: "web-proxy", + ProxyID: "web-proxy", + Address: "1.2.3.4", + Port: 8080, + Proxy: structs.ConnectProxyConfig{ + LocalServicePort: 8080, + Expose: structs.ExposeConfig{ + Checks: false, + Paths: []structs.ExposePath{ + { + LocalPathPort: 8080, + Path: "/health1", + ListenerPort: 21500, + }, + { + LocalPathPort: 8080, + Path: "/health2", + ListenerPort: 21501, + }, + }, + }, + }, + Datacenter: "dc1", + } +} + // ControllableCacheType is a cache.Type that simulates a typical blocking RPC // but lets us control the responses and when they are delivered easily. type ControllableCacheType struct { diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 44f3333f3528..ad5328f4d463 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -176,6 +176,22 @@ func TestClustersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotDiscoveryChain_SplitterWithResolverRedirectMultiDC, setup: nil, }, + { + name: "expose-paths-local-app-paths", + create: proxycfg.TestConfigSnapshotExposeConfig, + }, + { + name: "expose-paths-new-cluster-http2", + create: proxycfg.TestConfigSnapshotExposeConfig, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.Proxy.Expose.Paths[1] = structs.ExposePath{ + LocalPathPort: 9090, + Path: "/grpc.health.v1.Health/Check", + ListenerPort: 21501, + Protocol: "http2", + } + }, + }, { name: "mesh-gateway", create: proxycfg.TestConfigSnapshotMeshGateway, diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 5d48fae0fab9..f1a02453aaf8 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -204,6 +204,22 @@ func TestListenersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailoverThroughLocalGateway, setup: nil, }, + { + name: "expose-paths-local-app-paths", + create: proxycfg.TestConfigSnapshotExposeConfig, + }, + { + name: "expose-paths-new-cluster-http2", + create: proxycfg.TestConfigSnapshotExposeConfig, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.Proxy.Expose.Paths[1] = structs.ExposePath{ + LocalPathPort: 9090, + Path: "/grpc.health.v1.Health/Check", + ListenerPort: 21501, + Protocol: "http2", + } + }, + }, { name: "mesh-gateway", create: proxycfg.TestConfigSnapshotMeshGateway, diff --git a/agent/xds/testdata/clusters/expose-paths-local-app-paths.golden b/agent/xds/testdata/clusters/expose-paths-local-app-paths.golden new file mode 100644 index 000000000000..40fc33eab50f --- /dev/null +++ b/agent/xds/testdata/clusters/expose-paths-local-app-paths.golden @@ -0,0 +1,32 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/expose-paths-new-cluster-http2.golden b/agent/xds/testdata/clusters/expose-paths-new-cluster-http2.golden new file mode 100644 index 000000000000..8494c8e061ff --- /dev/null +++ b/agent/xds/testdata/clusters/expose-paths-new-cluster-http2.golden @@ -0,0 +1,60 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "exposed_cluster_9090", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "exposed_cluster_9090", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9090 + } + } + } + } + ] + } + ] + }, + "http2ProtocolOptions": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/expose-paths-local-app-paths.golden b/agent/xds/testdata/listeners/expose-paths-local-app-paths.golden new file mode 100644 index 000000000000..60a30df1f326 --- /dev/null +++ b/agent/xds/testdata/listeners/expose-paths-local-app-paths.golden @@ -0,0 +1,154 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_health1:1.2.3.4:21500", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21500 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_health1_21500", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_health1_21500", + "routes": [ + { + "match": { + "path": "/health1" + }, + "route": { + "cluster": "local_app" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_health1_21500_http", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_health2:1.2.3.4:21501", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21501 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_health2_21501", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_health2_21501", + "routes": [ + { + "match": { + "path": "/health2" + }, + "route": { + "cluster": "local_app" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_health2_21501_http", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "tlsContext": { + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.ext_authz", + "config": { + "grpc_service": { + "envoy_grpc": { + "cluster_name": "local_agent" + }, + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "my-token" + } + ] + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener_tcp" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden b/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden new file mode 100644 index 000000000000..633f3c529fab --- /dev/null +++ b/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden @@ -0,0 +1,157 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_grpchealthv1HealthCheck:1.2.3.4:21501", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21501 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "codec_type": "HTTP2", + "http2_protocol_options": { + }, + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_grpchealthv1HealthCheck_21501", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_grpchealthv1HealthCheck_21501", + "routes": [ + { + "match": { + "path": "/grpc.health.v1.Health/Check" + }, + "route": { + "cluster": "exposed_cluster_9090" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_grpchealthv1HealthCheck_21501_http", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "exposed_path_health1:1.2.3.4:21500", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 21500 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "route_config": { + "name": "exposed_path_filter_health1_21500", + "virtual_hosts": [ + { + "domains": [ + "*" + ], + "name": "exposed_path_filter_health1_21500", + "routes": [ + { + "match": { + "path": "/health1" + }, + "route": { + "cluster": "local_app" + } + } + ] + } + ] + }, + "stat_prefix": "exposed_path_filter_health1_21500_http", + "tracing": { + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "public_listener:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "tlsContext": { + "requireClientCertificate": true + }, + "filters": [ + { + "name": "envoy.ext_authz", + "config": { + "grpc_service": { + "envoy_grpc": { + "cluster_name": "local_agent" + }, + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "my-token" + } + ] + }, + "stat_prefix": "connect_authz" + } + }, + { + "name": "envoy.tcp_proxy", + "config": { + "cluster": "local_app", + "stat_prefix": "public_listener_tcp" + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file From c0e6a44ff4c4b129af0c1e79a89c3d554f33163e Mon Sep 17 00:00:00 2001 From: Freddy Date: Thu, 5 Sep 2019 07:43:31 -0600 Subject: [PATCH 46/80] Update agent/agent.go Co-Authored-By: Hans Hasselberg --- agent/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agent.go b/agent/agent.go index a8d6cff7e2e3..faacdace40ec 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3647,7 +3647,7 @@ func (a *Agent) resetExposedChecks(serviceID string) { // listenerPort allocates a port from the configured range // The agent stateLock MUST be held when this is called -func (a *Agent) listenerPort(svcID, checkID string) (int, error) { +func (a *Agent) listenerPortLocked(svcID, checkID string) (int, error) { key := fmt.Sprintf("%s:%s", svcID, checkID) if a.exposedPorts == nil { a.exposedPorts = make(map[string]int) From af5b5cabc2543df07e31c6363d5aa87d4e9d0bff Mon Sep 17 00:00:00 2001 From: Freddy Date: Thu, 5 Sep 2019 07:43:52 -0600 Subject: [PATCH 47/80] Update agent/agent.go Co-Authored-By: Hans Hasselberg --- agent/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agent.go b/agent/agent.go index faacdace40ec..3c0b1e1b0258 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3608,7 +3608,7 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string) error { // The only way to get here is if the regex pattern fails to compile, which would be caught by tests return fmt.Errorf("failed to inject proxy addr into HTTP target") } - c.ProxyHTTP = addr + c.ProxyHTTP = httpInjectAddr(c.HTTP, proxyAddr, port) } for _, c := range a.checkGRPCs { if c.ServiceID != serviceID { From 520c7c1f4348ed9ab7945fd7af6c0cdf02b35374 Mon Sep 17 00:00:00 2001 From: Freddy Date: Thu, 5 Sep 2019 07:44:04 -0600 Subject: [PATCH 48/80] Update agent/agent.go Co-Authored-By: Hans Hasselberg --- agent/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agent.go b/agent/agent.go index 3c0b1e1b0258..22af9ebb74dc 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3619,7 +3619,7 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string) error { return err } addr := grpcInjectAddr(c.GRPC, proxyAddr, port) - c.ProxyGRPC = addr + c.ProxyGRPC = grpcInjectAddr(c.GRPC, proxyAddr, port) } return nil } From 4c7d7e117b00122766732f5c13ad0418d815fda9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 07:56:32 -0600 Subject: [PATCH 49/80] PR comments --- agent/agent.go | 3 +-- agent/structs/check_type.go | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index a8d6cff7e2e3..e4059b0fa4f9 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2386,8 +2386,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, // Get the address of the proxy for this service if it exists // Need its config to know whether we should reroute checks to it var proxy *structs.NodeService - services := a.State.Services() - for _, svc := range services { + for _, svc := range a.State.Services() { if svc.Proxy.DestinationServiceID == service.ID { proxy = svc } diff --git a/agent/structs/check_type.go b/agent/structs/check_type.go index 1b217e2e91ab..a2282bdd11d7 100644 --- a/agent/structs/check_type.go +++ b/agent/structs/check_type.go @@ -13,6 +13,8 @@ import ( // HTTP, Docker, TCP and GRPC all require Interval. Only one of the types may // to be provided: TTL or Script/Interval or HTTP/Interval or TCP/Interval or // Docker/Interval or GRPC/Interval or AliasService. +// Since types like CheckHTTP and CheckGRPC derive from CheckType, there are +// helper conversion methods that do the reverse conversion. ie. checkHTTP.CheckType() type CheckType struct { // fields already embedded in CheckDefinition // Note: CheckType.CheckID == CheckDefinition.ID From 7357e7eae8ee60ba7bffefebce4df2fd48750319 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 08:13:18 -0600 Subject: [PATCH 50/80] Resolve conflicts --- agent/agent.go | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index edc2b69f4dea..1f66eb8d784b 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2446,13 +2446,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } if proxy != nil && proxy.Proxy.Expose.Checks { - port, err := a.listenerPort(service.ID, string(http.CheckID)) + port, err := a.listenerPortLocked(service.ID, string(http.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) return err } - addr := httpInjectAddr(http.HTTP, proxy.Proxy.LocalServiceAddress, port) - http.ProxyHTTP = addr + http.ProxyHTTP = httpInjectAddr(http.HTTP, proxy.Proxy.LocalServiceAddress, port) } http.Start() @@ -2509,13 +2508,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } if proxy != nil && proxy.Proxy.Expose.Checks { - port, err := a.listenerPort(service.ID, string(grpc.CheckID)) + port, err := a.listenerPortLocked(service.ID, string(grpc.CheckID)) if err != nil { a.logger.Printf("[ERR] agent: error exposing check: %s", err) return err } - addr := grpcInjectAddr(grpc.GRPC, proxy.Proxy.LocalServiceAddress, port) - grpc.ProxyGRPC = addr + grpc.ProxyGRPC = grpcInjectAddr(grpc.GRPC, proxy.Proxy.LocalServiceAddress, port) } grpc.Start() @@ -3598,26 +3596,20 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string) error { if c.ServiceID != serviceID { continue } - port, err := a.listenerPort(serviceID, string(c.CheckID)) + port, err := a.listenerPortLocked(serviceID, string(c.CheckID)) if err != nil { return err } - addr := httpInjectAddr(c.HTTP, proxyAddr, port) - if err != nil { - // The only way to get here is if the regex pattern fails to compile, which would be caught by tests - return fmt.Errorf("failed to inject proxy addr into HTTP target") - } c.ProxyHTTP = httpInjectAddr(c.HTTP, proxyAddr, port) } for _, c := range a.checkGRPCs { if c.ServiceID != serviceID { continue } - port, err := a.listenerPort(serviceID, string(c.CheckID)) + port, err := a.listenerPortLocked(serviceID, string(c.CheckID)) if err != nil { return err } - addr := grpcInjectAddr(c.GRPC, proxyAddr, port) c.ProxyGRPC = grpcInjectAddr(c.GRPC, proxyAddr, port) } return nil From bc0709e140da3fd151ec7784813ba8ffe55796a9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 08:28:22 -0600 Subject: [PATCH 51/80] More PR review comments --- agent/agent.go | 21 ++++++++++++++------- agent/agent_test.go | 18 +++++++++--------- agent/cache-types/service_checks.go | 10 +++++----- agent/cache-types/service_checks_test.go | 2 +- agent/service_checks_test.go | 2 +- agent/xds/clusters.go | 2 +- agent/xds/listeners.go | 2 +- agent/xds/server.go | 2 +- 8 files changed, 33 insertions(+), 26 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 1f66eb8d784b..58a86ec8eb3a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2389,6 +2389,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, for _, svc := range a.State.Services() { if svc.Proxy.DestinationServiceID == service.ID { proxy = svc + break } } @@ -2661,6 +2662,7 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { for _, c := range a.State.Checks() { if c.CheckID == checkID { svcID = c.ServiceID + break } } s := a.State.ServiceState(svcID) @@ -2673,7 +2675,7 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { // Delete port from allocated port set // If checks weren't being exposed then this is a no-op - portKey := fmt.Sprintf("%s:%s", svcID, checkID) + portKey := listenerPortKey(svcID, string(checkID)) delete(a.exposedPorts, portKey) a.cancelCheckMonitors(checkID) @@ -2692,7 +2694,7 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { return nil } -func (a *Agent) ServiceHTTPChecks(serviceID string) []structs.CheckType { +func (a *Agent) ServiceHTTPBasedChecks(serviceID string) []structs.CheckType { var chkTypes = make([]structs.CheckType, 0) for _, c := range a.checkHTTPs { if c.ServiceID == serviceID { @@ -3619,27 +3621,28 @@ func (a *Agent) rerouteExposedChecks(serviceID string, proxyAddr string) error { // Future calls to check() will use the original target c.HTTP or c.GRPC // The agent stateLock MUST be held for this to be called func (a *Agent) resetExposedChecks(serviceID string) { + ids := make([]string, 0) for _, c := range a.checkHTTPs { if c.ServiceID == serviceID { c.ProxyHTTP = "" + ids = append(ids, string(c.CheckID)) } } for _, c := range a.checkGRPCs { if c.ServiceID == serviceID { c.ProxyGRPC = "" + ids = append(ids, string(c.CheckID)) } } - for k, _ := range a.exposedPorts { - if strings.HasPrefix(k, serviceID) { - delete(a.exposedPorts, k) - } + for _, checkID := range ids { + delete(a.exposedPorts, listenerPortKey(serviceID, checkID)) } } // listenerPort allocates a port from the configured range // The agent stateLock MUST be held when this is called func (a *Agent) listenerPortLocked(svcID, checkID string) (int, error) { - key := fmt.Sprintf("%s:%s", svcID, checkID) + key := listenerPortKey(svcID, checkID) if a.exposedPorts == nil { a.exposedPorts = make(map[string]int) } @@ -3667,6 +3670,10 @@ func (a *Agent) listenerPortLocked(svcID, checkID string) (int, error) { return port, nil } +func listenerPortKey(svcID, checkID string) string { + return fmt.Sprintf("%s:%s", svcID, checkID) +} + // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] func grpcInjectAddr(existing string, ip string, port int) string { r := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") diff --git a/agent/agent_test.go b/agent/agent_test.go index 7e9d7595955c..7914dd69ea70 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3604,7 +3604,7 @@ func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { } retry.Run(t, func(r *retry.R) { - chks := a.ServiceHTTPChecks("web") + chks := a.ServiceHTTPBasedChecks("web") got := chks[0].ProxyHTTP if got == "" { @@ -3618,9 +3618,9 @@ func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { }) retry.Run(t, func(r *retry.R) { - chks := a.ServiceHTTPChecks("web") + chks := a.ServiceHTTPBasedChecks("web") - // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPChecks + // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPBasedChecks got := chks[1].ProxyGRPC if got == "" { r.Fatal("ProxyGRPC addr not set in check") @@ -3656,7 +3656,7 @@ func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { } retry.Run(t, func(r *retry.R) { - chks := a.ServiceHTTPChecks("web") + chks := a.ServiceHTTPBasedChecks("web") got := chks[0].ProxyHTTP if got != "" { @@ -3665,9 +3665,9 @@ func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { }) retry.Run(t, func(r *retry.R) { - chks := a.ServiceHTTPChecks("web") + chks := a.ServiceHTTPBasedChecks("web") - // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPChecks + // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPBasedChecks got := chks[1].ProxyGRPC if got != "" { r.Fatal("ProxyGRPC addr was not reset") @@ -3755,7 +3755,7 @@ func TestAgent_RerouteNewHTTPChecks(t *testing.T) { } retry.Run(t, func(r *retry.R) { - chks := a.ServiceHTTPChecks("web") + chks := a.ServiceHTTPBasedChecks("web") got := chks[0].ProxyHTTP if got == "" { @@ -3769,9 +3769,9 @@ func TestAgent_RerouteNewHTTPChecks(t *testing.T) { }) retry.Run(t, func(r *retry.R) { - chks := a.ServiceHTTPChecks("web") + chks := a.ServiceHTTPBasedChecks("web") - // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPChecks + // Will be at a later index than HTTP check because of the fetching order in ServiceHTTPBasedChecks got := chks[1].ProxyGRPC if got == "" { r.Fatal("ProxyGRPC addr not set in check") diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go index 2ddae4ad68a7..5bb60681f79b 100644 --- a/agent/cache-types/service_checks.go +++ b/agent/cache-types/service_checks.go @@ -16,12 +16,12 @@ import ( const ServiceHTTPChecksName = "service-http-checks" type Agent interface { - ServiceHTTPChecks(id string) []structs.CheckType + ServiceHTTPBasedChecks(id string) []structs.CheckType LocalState() *local.State SyncPausedCh() <-chan struct{} } -// ServiceHTTPChecks supports fetching discovering checks in the local state +// ServiceHTTPBasedChecks supports fetching discovering checks in the local state type ServiceHTTPChecks struct { Agent Agent } @@ -81,7 +81,7 @@ WATCH_LOOP: // WatchCh will receive updates on service (de)registrations and check (de)registrations ws.Add(svcState.WatchCh) - resp = c.Agent.ServiceHTTPChecks(reqReal.ServiceID) + resp = c.Agent.ServiceHTTPBasedChecks(reqReal.ServiceID) hash, err = hashChecks(resp) if err != nil { @@ -94,7 +94,7 @@ WATCH_LOOP: } // Watch returned false indicating a change was detected, loop and repeat - // the call to ServiceHTTPChecks to load the new value. + // the call to ServiceHTTPBasedChecks to load the new value. // If agent sync is paused it means local state is being bulk-edited e.g. config reload. if syncPauseCh := c.Agent.SyncPausedCh(); syncPauseCh != nil { // Wait for pause to end or for the timeout to elapse. @@ -126,7 +126,7 @@ func (c *ServiceHTTPChecks) SupportsBlocking() bool { } // ServiceHTTPChecksRequest is the cache.Request implementation for the -// ServiceHTTPChecks cache type. This is implemented here and not in structs +// ServiceHTTPBasedChecks cache type. This is implemented here and not in structs // since this is only used for cache-related requests and not forwarded // directly to any Consul servers. type ServiceHTTPChecksRequest struct { diff --git a/agent/cache-types/service_checks_test.go b/agent/cache-types/service_checks_test.go index 133a80cb9a45..031fcc7bd43b 100644 --- a/agent/cache-types/service_checks_test.go +++ b/agent/cache-types/service_checks_test.go @@ -149,7 +149,7 @@ func newMockAgent() *mockAgent { return &m } -func (m *mockAgent) ServiceHTTPChecks(id string) []structs.CheckType { +func (m *mockAgent) ServiceHTTPBasedChecks(id string) []structs.CheckType { return m.checks } diff --git a/agent/service_checks_test.go b/agent/service_checks_test.go index b4f45ab3b5ec..bdad4b978f4c 100644 --- a/agent/service_checks_test.go +++ b/agent/service_checks_test.go @@ -12,7 +12,7 @@ import ( "time" ) -// Integration test for ServiceHTTPChecks cache-type +// Integration test for ServiceHTTPBasedChecks cache-type // Placed in agent pkg rather than cache-types to avoid circular dependency when importing agent.TestAgent func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { t.Parallel() diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index bb2e93aa5a47..38479f1979be 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -79,7 +79,7 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh // Add service health checks to the list of paths to create clusters for if needed if cfgSnap.Proxy.Expose.Checks { - for _, check := range s.CheckFetcher.ServiceHTTPChecks(cfgSnap.Proxy.DestinationServiceID) { + for _, check := range s.CheckFetcher.ServiceHTTPBasedChecks(cfgSnap.Proxy.DestinationServiceID) { p, err := parseCheckPath(check) if err != nil { s.Logger.Printf("[WARN] envoy: failed to create cluster for check '%s': %v", check.CheckID, err) diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index ef006259634f..53b908411737 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -79,7 +79,7 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps // Add service health checks to the list of paths to create listeners for if needed if cfgSnap.Proxy.Expose.Checks { - for _, check := range s.CheckFetcher.ServiceHTTPChecks(cfgSnap.Proxy.DestinationServiceID) { + for _, check := range s.CheckFetcher.ServiceHTTPBasedChecks(cfgSnap.Proxy.DestinationServiceID) { p, err := parseCheckPath(check) if err != nil { s.Logger.Printf("[WARN] envoy: failed to create listener for check '%s': %v", check.CheckID, err) diff --git a/agent/xds/server.go b/agent/xds/server.go index 1cc346dba044..33d22621c57e 100644 --- a/agent/xds/server.go +++ b/agent/xds/server.go @@ -100,7 +100,7 @@ type ConnectAuthz interface { // ServiceChecks is the interface the agent needs to expose // for the xDS server to fetch a service's HTTP check definitions type HTTPCheckFetcher interface { - ServiceHTTPChecks(serviceID string) []structs.CheckType + ServiceHTTPBasedChecks(serviceID string) []structs.CheckType } // ConfigManager is the interface xds.Server requires to consume proxy config From 0ed1b8f7aa4049678ed71aad56b24607f6599d0a Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 12:26:34 -0600 Subject: [PATCH 52/80] More PR review comments, like parsing url with url.Parse() --- agent/agent_test.go | 1 + agent/structs/structs.go | 9 +++++-- agent/xds/listeners.go | 58 ++++++++++++++++++++++++++-------------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index 7914dd69ea70..9619a8fbdcf1 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3626,6 +3626,7 @@ func TestAgent_RerouteExistingHTTPChecks(t *testing.T) { r.Fatal("ProxyGRPC addr not set in check") } + // Node that this relies on listener ports auto-incrementing in a.listenerPortLocked want := "localhost:21501/myservice" if got != want { r.Fatalf("unexpected proxy addr in check, want: %s, got: %s", want, got) diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 145b53fe69bc..d076a4d532bb 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -971,9 +971,14 @@ func (s *NodeService) Validate() error { path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { + protocols := make([]string, 0) + for p, _ := range allowedExposeProtocols { + protocols = append(protocols, p) + } + result = multierror.Append(result, - fmt.Errorf("protocol '%s' not supported for path: %s, must be http or http2", - path.Protocol, path.Path)) + fmt.Errorf("protocol '%s' not supported for path: %s, must be in: %v", + path.Protocol, path.Path, protocols)) } } } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 53b908411737..e62c8fcb896b 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -4,6 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "net" + "net/url" "regexp" "strconv" "strings" @@ -107,28 +109,38 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps } func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { - grpcRE := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") - httpRE := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) - var path structs.ExposePath - var err error if check.HTTP != "" { path.Protocol = "http" - matches := httpRE.FindStringSubmatch(check.HTTP) - path.Path = matches[4] + // Get path and local port from original HTTP target + u, err := url.Parse(check.HTTP) + if err != nil { + return path, fmt.Errorf("failed to parse url '%s': %v", check.HTTP, err) + } + path.Path = u.Path - localStr := strings.TrimPrefix(matches[3], ":") - path.LocalPathPort, err = strconv.Atoi(localStr) + _, portStr, err := net.SplitHostPort(u.Host) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.HTTP, err) + } + path.LocalPathPort, err = strconv.Atoi(portStr) if err != nil { return path, fmt.Errorf("failed to parse port from '%s': %v", check.HTTP, err) } - matches = httpRE.FindStringSubmatch(check.ProxyHTTP) + // Get listener port from proxied HTTP target + u, err = url.Parse(check.ProxyHTTP) + if err != nil { + return path, fmt.Errorf("failed to parse url '%s': %v", check.ProxyHTTP, err) + } - listenerStr := strings.TrimPrefix(matches[3], ":") - path.ListenerPort, err = strconv.Atoi(listenerStr) + _, portStr, err = net.SplitHostPort(u.Host) + if err != nil { + return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyHTTP, err) + } + path.ListenerPort, err = strconv.Atoi(portStr) if err != nil { return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyHTTP, err) } @@ -138,18 +150,24 @@ func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { path.Path = "/grpc.health.v1.Health/Check" path.Protocol = "http2" - matches := grpcRE.FindStringSubmatch(check.GRPC) - - localStr := strings.TrimPrefix(matches[2], ":") - path.LocalPathPort, err = strconv.Atoi(localStr) + // Get local port from original GRPC target of the form: host/service + proxyServerAndService := strings.SplitN(check.GRPC, "/", 2) + _, portStr, err := net.SplitHostPort(proxyServerAndService[0]) + if err != nil { + return path, fmt.Errorf("failed to split host/port from '%s': %v", check.GRPC, err) + } + path.LocalPathPort, err = strconv.Atoi(portStr) if err != nil { return path, fmt.Errorf("failed to parse port from '%s': %v", check.GRPC, err) } - matches = grpcRE.FindStringSubmatch(check.ProxyGRPC) - - listenerStr := strings.TrimPrefix(matches[2], ":") - path.ListenerPort, err = strconv.Atoi(listenerStr) + // Get listener port from proxied GRPC target of the form: host/service + proxyServerAndService = strings.SplitN(check.ProxyGRPC, "/", 2) + _, portStr, err = net.SplitHostPort(proxyServerAndService[0]) + if err != nil { + return path, fmt.Errorf("failed to split host/port from '%s': %v", check.ProxyGRPC, err) + } + path.ListenerPort, err = strconv.Atoi(portStr) if err != nil { return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyGRPC, err) } @@ -372,7 +390,7 @@ func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, clus addr = "0.0.0.0" } - // Strip any special characters from path + // Strip any special characters from path to make a valid and hopefully unique name r := regexp.MustCompile(`[^a-zA-Z0-9]+`) strippedPath := r.ReplaceAllString(path, "") listenerName := fmt.Sprintf("exposed_path_%s", strippedPath) From bbfbbdd6eb72d8720576b509187784c3b2dbc01c Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 12:30:51 -0600 Subject: [PATCH 53/80] Return error on duplicate exposed listener ports --- agent/structs/structs.go | 12 +++++++++--- agent/structs/structs_test.go | 7 +++++++ agent/structs/testing_catalog.go | 8 ++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/agent/structs/structs.go b/agent/structs/structs.go index d076a4d532bb..05343b01fbca 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -947,16 +947,22 @@ func (s *NodeService) Validate() error { } bindAddrs[addr] = struct{}{} } - var known = make(map[string]bool) + var knownPaths = make(map[string]bool) + var knownListeners = make(map[int]bool) for _, path := range s.Proxy.Expose.Paths { if path.Path == "" { result = multierror.Append(result, fmt.Errorf("empty path exposed")) } - if seen := known[path.Path]; seen { + if seen := knownPaths[path.Path]; seen { result = multierror.Append(result, fmt.Errorf("duplicate paths exposed")) } - known[path.Path] = true + knownPaths[path.Path] = true + + if seen := knownListeners[path.ListenerPort]; seen { + result = multierror.Append(result, fmt.Errorf("duplicate listener ports exposed")) + } + knownListeners[path.ListenerPort] = true if path.ListenerPort <= 0 || path.ListenerPort > 65535 { result = multierror.Append(result, fmt.Errorf("invalid listener port: %d", path.ListenerPort)) diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index 333cdcd7ff19..de2aee54641e 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -443,6 +443,13 @@ func TestStructs_NodeService_ValidateExposeConfig(t *testing.T) { }, "duplicate paths exposed", }, + "duplicate ports": { + func(x *NodeService) { + x.Proxy.Expose.Paths[0].ListenerPort = 21600 + x.Proxy.Expose.Paths[1].ListenerPort = 21600 + }, + "duplicate listener ports exposed", + }, "protocol not supported": { func(x *NodeService) { x.Proxy.Expose.Paths[0].Protocol = "foo" }, "protocol 'foo' not supported for path", diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index ddc5fb4efbf3..f25c4ef5bd07 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -62,13 +62,13 @@ func TestNodeServiceExpose(t testing.T) *NodeService { Paths: []ExposePath{ { Path: "/foo", - LocalPathPort: 80, - ListenerPort: 80, + LocalPathPort: 8080, + ListenerPort: 21500, }, { Path: "/bar", - LocalPathPort: 80, - ListenerPort: 80, + LocalPathPort: 8080, + ListenerPort: 21501, }, }, }, From 6c8744572608140e8585920e95b03fad8c2f2a50 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 12:55:05 -0600 Subject: [PATCH 54/80] Use set tag when hashing slice --- agent/cache-types/service_checks.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go index 5bb60681f79b..5be254f0c781 100644 --- a/agent/cache-types/service_checks.go +++ b/agent/cache-types/service_checks.go @@ -8,7 +8,6 @@ import ( "github.com/hashicorp/consul/lib" "github.com/hashicorp/go-memdb" "github.com/mitchellh/hashstructure" - "strings" "time" ) @@ -146,13 +145,16 @@ func (s *ServiceHTTPChecksRequest) CacheInfo() cache.RequestInfo { } func hashChecks(checks []structs.CheckType) (string, error) { - var b strings.Builder - for _, check := range checks { - raw, err := hashstructure.Hash(check, nil) - if err != nil { - return "", fmt.Errorf("failed to hash check '%s': %v", check.CheckID, err) - } - fmt.Fprintf(&b, "%x", raw) + // Wrapper created to use "set" struct tag, that way ordering doesn't lead to false-positives + wrapper := struct { + ChkTypes []structs.CheckType `hash:"set"` + }{ + ChkTypes: checks, + } + + b, err := hashstructure.Hash(wrapper, nil) + if err != nil { + return "", fmt.Errorf("failed to hash checks: %v", err) } - return b.String(), nil + return fmt.Sprintf("%d", b), nil } From f4df424b363001730e625740e8a6359d109da45e Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 5 Sep 2019 15:47:00 -0600 Subject: [PATCH 55/80] Improve cache type test --- agent/cache-types/service_checks.go | 4 +++ agent/cache-types/service_checks_test.go | 31 +++++++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go index 5be254f0c781..96ae286574ef 100644 --- a/agent/cache-types/service_checks.go +++ b/agent/cache-types/service_checks.go @@ -145,6 +145,10 @@ func (s *ServiceHTTPChecksRequest) CacheInfo() cache.RequestInfo { } func hashChecks(checks []structs.CheckType) (string, error) { + if len(checks) == 0 { + return "", nil + } + // Wrapper created to use "set" struct tag, that way ordering doesn't lead to false-positives wrapper := struct { ChkTypes []structs.CheckType `hash:"set"` diff --git a/agent/cache-types/service_checks_test.go b/agent/cache-types/service_checks_test.go index 031fcc7bd43b..193e12b36c2b 100644 --- a/agent/cache-types/service_checks_test.go +++ b/agent/cache-types/service_checks_test.go @@ -26,6 +26,10 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { GRPC: "localhost:9090/v1.Health", Interval: 5 * time.Second, }, + { + CheckID: "ttl-check", + TTL: 10 * time.Second, + }, } svcState := local.ServiceState{ @@ -39,11 +43,28 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { a.LocalState().SetServiceState(&svcState) typ := cachetype.ServiceHTTPChecks{Agent: a} + // Adding TTL check should not yield result from Fetch since TTL checks aren't tracked + if err := a.AddCheck(*chkTypes[2]); err != nil { + t.Fatalf("failed to add check: %v", err) + } + result, err := typ.Fetch( + cache.FetchOptions{}, + &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, + ) + if err != nil { + t.Fatalf("failed to fetch: %v", err) + } + got, ok := result.Value.(*[]structs.CheckType) + if !ok { + t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + } + require.Empty(t, *got) + // Adding HTTP check should yield check in Fetch if err := a.AddCheck(*chkTypes[0]); err != nil { t.Fatalf("failed to add check: %v", err) } - result, err := typ.Fetch( + result, err = typ.Fetch( cache.FetchOptions{}, &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, ) @@ -51,10 +72,10 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { t.Fatalf("failed to fetch: %v", err) } if result.Index != 1 { - t.Fatalf("expected index of 1 after first request, got %d", result.Index) + t.Fatalf("expected index of 1 after first cache hit, got %d", result.Index) } - got, ok := result.Value.(*[]structs.CheckType) + got, ok = result.Value.(*[]structs.CheckType) if !ok { t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) } @@ -162,7 +183,9 @@ func (m *mockAgent) SyncPausedCh() <-chan struct{} { } func (m *mockAgent) AddCheck(check structs.CheckType) error { - m.checks = append(m.checks, check) + if check.IsHTTP() || check.IsGRPC() { + m.checks = append(m.checks, check) + } return nil } From 3a6a52602fc268567306d13aea20ed7ec098c82e Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 11 Sep 2019 23:49:24 -0700 Subject: [PATCH 56/80] Expand error msgs on NodeService validation --- agent/structs/structs.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 05343b01fbca..bc1c8787ea3d 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -951,29 +951,32 @@ func (s *NodeService) Validate() error { var knownListeners = make(map[int]bool) for _, path := range s.Proxy.Expose.Paths { if path.Path == "" { - result = multierror.Append(result, fmt.Errorf("empty path exposed")) + result = multierror.Append(result, fmt.Errorf("expose.paths: empty path exposed")) } if seen := knownPaths[path.Path]; seen { - result = multierror.Append(result, fmt.Errorf("duplicate paths exposed")) + result = multierror.Append(result, fmt.Errorf("expose.paths: duplicate paths exposed")) } knownPaths[path.Path] = true if seen := knownListeners[path.ListenerPort]; seen { - result = multierror.Append(result, fmt.Errorf("duplicate listener ports exposed")) + result = multierror.Append(result, fmt.Errorf("expose.paths: duplicate listener ports exposed")) } knownListeners[path.ListenerPort] = true if path.ListenerPort <= 0 || path.ListenerPort > 65535 { - result = multierror.Append(result, fmt.Errorf("invalid listener port: %d", path.ListenerPort)) + result = multierror.Append(result, fmt.Errorf("expose.paths: invalid listener port: %d", path.ListenerPort)) } if path.CAFile != "" { _, err := os.Stat(path.CAFile) if err != nil { - result = multierror.Append(result, fmt.Errorf("failed to find CAFile '%s': %v", path.CAFile, err)) + result = multierror.Append(result, fmt.Errorf("expose.paths: failed to find CAFile '%s': %v", path.CAFile, err)) } } + if path.TLSSkipVerify == false && path.CAFile == "" { + result = multierror.Append(result, fmt.Errorf("expose.paths: CAFile must be provided if tls_skip_verify is false")) + } path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { From 5a02b07f923c50e5459a742d368ccdcd81de145b Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 12 Sep 2019 00:02:23 -0700 Subject: [PATCH 57/80] Add badreq test service checks cache-type --- agent/cache-types/service_checks_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/agent/cache-types/service_checks_test.go b/agent/cache-types/service_checks_test.go index 193e12b36c2b..5b204d55f5d5 100644 --- a/agent/cache-types/service_checks_test.go +++ b/agent/cache-types/service_checks_test.go @@ -154,6 +154,17 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { } } +func TestServiceHTTPChecks_badReqType(t *testing.T) { + a := newMockAgent() + typ := cachetype.ServiceHTTPChecks{Agent: a} + + // Fetch + _, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest( + t, cache.RequestInfo{Key: "foo", MinIndex: 64})) + require.Error(t, err) + require.Contains(t, err.Error(), "wrong request type") +} + type mockAgent struct { state *local.State pauseCh <-chan struct{} From 5a53d6795a13e2b8d679c3b343fed6e692e1a825 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 17 Sep 2019 16:08:11 -0600 Subject: [PATCH 58/80] Move localBlockingQuery to Agent --- agent/agent.go | 67 ++++++++++++++++++++ agent/agent_endpoint.go | 77 +++-------------------- agent/cache-types/service_checks.go | 78 +++++++----------------- agent/cache-types/service_checks_test.go | 78 +++++++++++++----------- agent/event_endpoint.go | 5 -- agent/proxycfg/state.go | 6 +- agent/service_checks_test.go | 19 +++--- 7 files changed, 152 insertions(+), 178 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 58a86ec8eb3a..25404716d216 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "github.com/hashicorp/go-memdb" "io" "io/ioutil" "log" @@ -77,6 +78,13 @@ const ( // ID of the leaf watch leafWatchID = "leaf" + + // maxQueryTime is used to bound the limit of a blocking query + maxQueryTime = 600 * time.Second + + // defaultQueryTime is the amount of time we block waiting for a change + // if no time is specified. Previously we would wait the maxQueryTime. + defaultQueryTime = 300 * time.Second ) type configSource int @@ -3455,6 +3463,65 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error { return nil } +// LocalBlockingQuery performs a blocking query in a generic way against +// local agent state that has no RPC or raft to back it. It uses `hash` parameter +// instead of an `index`. +// `alwaysBlock` determines whether we block if the provided hash is empty. +// Callers like the AgentService endpoint will want to return the current result if a hash isn't provided. +// On the other hand, for cache notifications we always want to block. This avoids an empty first response. +func (a *Agent) LocalBlockingQuery(alwaysBlock bool, hash string, wait time.Duration, + fn func(ws memdb.WatchSet) (string, interface{}, error)) (string, interface{}, error) { + + // If we are not blocking we can skip tracking and allocating - nil WatchSet + // is still valid to call Add on and will just be a no op. + var ws memdb.WatchSet + var timeout *time.Timer + + if alwaysBlock || hash != "" { + if wait == 0 { + wait = defaultQueryTime + } + if wait > 10*time.Minute { + wait = maxQueryTime + } + // Apply a small amount of jitter to the request. + wait += lib.RandomStagger(wait / 16) + timeout = time.NewTimer(wait) + } + + for { + // Must reset this every loop in case the Watch set is already closed but + // hash remains same. In that case we'll need to re-block on ws.Watch() + // again. + ws = memdb.NewWatchSet() + curHash, curResp, err := fn(ws) + if err != nil { + return "", curResp, err + } + + // Return immediately if there is no timeout, the hash is different or the + // Watch returns true (indicating timeout fired). Note that Watch on a nil + // WatchSet immediately returns false which would incorrectly cause this to + // loop and repeat again, however we rely on the invariant that ws == nil + // IFF timeout == nil in which case the Watch call is never invoked. + if timeout == nil || hash != curHash || ws.Watch(timeout.C) { + return curHash, curResp, err + } + // Watch returned false indicating a change was detected, loop and repeat + // the callback to load the new value. If agent sync is paused it means + // local state is currently being bulk-edited e.g. config reload. In this + // case it's likely that local state just got unloaded and may or may not be + // reloaded yet. Wait a short amount of time for Sync to resume to ride out + // typical config reloads. + if syncPauseCh := a.SyncPausedCh(); syncPauseCh != nil { + select { + case <-syncPauseCh: + case <-timeout.C: + } + } + } +} + // registerCache configures the cache and registers all the supported // types onto the cache. This is NOT safe to call multiple times so // care should be taken to call this exactly once after the cache diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 78a8df6a69d2..63908128a588 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -3,15 +3,13 @@ package agent import ( "encoding/json" "fmt" + "github.com/hashicorp/go-memdb" + "github.com/mitchellh/hashstructure" "log" "net/http" "path/filepath" "strconv" "strings" - "time" - - "github.com/hashicorp/go-memdb" - "github.com/mitchellh/hashstructure" "github.com/hashicorp/consul/acl" cachetype "github.com/hashicorp/consul/agent/cache-types" @@ -262,7 +260,7 @@ func (s *HTTPServer) AgentService(resp http.ResponseWriter, req *http.Request) ( // in QueryOptions but I didn't want to make very general changes right away. hash := req.URL.Query().Get("hash") - return s.agentLocalBlockingQuery(resp, hash, &queryOpts, + resultHash, service, err := s.agent.LocalBlockingQuery(false, hash, queryOpts.MaxQueryTime, func(ws memdb.WatchSet) (string, interface{}, error) { svcState := s.agent.State.ServiceState(id) @@ -299,7 +297,12 @@ func (s *HTTPServer) AgentService(resp http.ResponseWriter, req *http.Request) ( reply.ContentHash = fmt.Sprintf("%x", rawHash) return reply.ContentHash, reply, nil - }) + }, + ) + if resultHash != "" { + resp.Header().Set("X-Consul-ContentHash", resultHash) + } + return service, err } func (s *HTTPServer) AgentChecks(resp http.ResponseWriter, req *http.Request) (interface{}, error) { @@ -1296,68 +1299,6 @@ func (s *HTTPServer) AgentConnectCALeafCert(resp http.ResponseWriter, req *http. return reply, nil } -type agentLocalBlockingFunc func(ws memdb.WatchSet) (string, interface{}, error) - -// agentLocalBlockingQuery performs a blocking query in a generic way against -// local agent state that has no RPC or raft to back it. It uses `hash` parameter -// instead of an `index`. The resp is needed to write the `X-Consul-ContentHash` -// header back on return no Status nor body content is ever written to it. -func (s *HTTPServer) agentLocalBlockingQuery(resp http.ResponseWriter, hash string, - queryOpts *structs.QueryOptions, fn agentLocalBlockingFunc) (interface{}, error) { - - // If we are not blocking we can skip tracking and allocating - nil WatchSet - // is still valid to call Add on and will just be a no op. - var ws memdb.WatchSet - var timeout *time.Timer - - if hash != "" { - // TODO(banks) at least define these defaults somewhere in a const. Would be - // nice not to duplicate the ones in consul/rpc.go too... - wait := queryOpts.MaxQueryTime - if wait == 0 { - wait = 5 * time.Minute - } - if wait > 10*time.Minute { - wait = 10 * time.Minute - } - // Apply a small amount of jitter to the request. - wait += lib.RandomStagger(wait / 16) - timeout = time.NewTimer(wait) - } - - for { - // Must reset this every loop in case the Watch set is already closed but - // hash remains same. In that case we'll need to re-block on ws.Watch() - // again. - ws = memdb.NewWatchSet() - curHash, curResp, err := fn(ws) - if err != nil { - return curResp, err - } - // Return immediately if there is no timeout, the hash is different or the - // Watch returns true (indicating timeout fired). Note that Watch on a nil - // WatchSet immediately returns false which would incorrectly cause this to - // loop and repeat again, however we rely on the invariant that ws == nil - // IFF timeout == nil in which case the Watch call is never invoked. - if timeout == nil || hash != curHash || ws.Watch(timeout.C) { - resp.Header().Set("X-Consul-ContentHash", curHash) - return curResp, err - } - // Watch returned false indicating a change was detected, loop and repeat - // the callback to load the new value. If agent sync is paused it means - // local state is currently being bulk-edited e.g. config reload. In this - // case it's likely that local state just got unloaded and may or may not be - // reloaded yet. Wait a short amount of time for Sync to resume to ride out - // typical config reloads. - if syncPauseCh := s.agent.SyncPausedCh(); syncPauseCh != nil { - select { - case <-syncPauseCh: - case <-timeout.C: - } - } - } -} - // AgentConnectAuthorize // // POST /v1/agent/connect/authorize diff --git a/agent/cache-types/service_checks.go b/agent/cache-types/service_checks.go index 96ae286574ef..72a0d8d782c8 100644 --- a/agent/cache-types/service_checks.go +++ b/agent/cache-types/service_checks.go @@ -5,7 +5,6 @@ import ( "github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/local" "github.com/hashicorp/consul/agent/structs" - "github.com/hashicorp/consul/lib" "github.com/hashicorp/go-memdb" "github.com/mitchellh/hashstructure" "time" @@ -17,7 +16,8 @@ const ServiceHTTPChecksName = "service-http-checks" type Agent interface { ServiceHTTPBasedChecks(id string) []structs.CheckType LocalState() *local.State - SyncPausedCh() <-chan struct{} + LocalBlockingQuery(alwaysBlock bool, hash string, wait time.Duration, + fn func(ws memdb.WatchSet) (string, interface{}, error)) (string, interface{}, error) } // ServiceHTTPBasedChecks supports fetching discovering checks in the local state @@ -35,77 +35,45 @@ func (c *ServiceHTTPChecks) Fetch(opts cache.FetchOptions, req cache.Request) (c "Internal cache failure: got wrong request type: %T, want: ServiceHTTPChecksRequest", req) } - var lastChecks *[]structs.CheckType + var lastChecks []structs.CheckType var lastHash string var err error // Hash last known result as a baseline if opts.LastResult != nil { - lastChecks, ok = opts.LastResult.Value.(*[]structs.CheckType) + lastChecks, ok = opts.LastResult.Value.([]structs.CheckType) if !ok { return result, fmt.Errorf( - "Internal cache failure: got wrong request type: %T, want: ServiceHTTPChecksRequest", req) + "Internal cache failure: last value in cache of wrong type: %T, want: CheckType", req) } - lastHash, err = hashChecks(*lastChecks) + lastHash, err = hashChecks(lastChecks) if err != nil { return result, fmt.Errorf("Internal cache failure: %v", err) } } - var wait time.Duration - - // Adjust wait based on documented limits and add some jitter: https://www.consul.io/api/features/blocking.html - switch wait = reqReal.MaxQueryTime; { - case wait == 0*time.Second: - wait = 5 * time.Minute - case wait > 10*time.Minute: - wait = 10 * time.Minute - } - timeout := time.NewTimer(wait + lib.RandomStagger(wait/16)) - - var resp []structs.CheckType - var hash string - -WATCH_LOOP: - for { - // Must reset this every loop in case the Watch set is already closed but - // hash remains same. In that case we'll need to re-block on ws.Watch() - ws := memdb.NewWatchSet() - - svcState := c.Agent.LocalState().ServiceState(reqReal.ServiceID) - if svcState == nil { - return result, fmt.Errorf("Internal cache failure: service '%s' not in agent state", reqReal.ServiceID) - } - - // WatchCh will receive updates on service (de)registrations and check (de)registrations - ws.Add(svcState.WatchCh) - - resp = c.Agent.ServiceHTTPBasedChecks(reqReal.ServiceID) + hash, resp, err := c.Agent.LocalBlockingQuery(true, lastHash, reqReal.MaxQueryTime, + func(ws memdb.WatchSet) (string, interface{}, error) { + svcState := c.Agent.LocalState().ServiceState(reqReal.ServiceID) + if svcState == nil { + return "", result, fmt.Errorf("Internal cache failure: service '%s' not in agent state", reqReal.ServiceID) + } - hash, err = hashChecks(resp) - if err != nil { - return result, fmt.Errorf("Internal cache failure: %v", err) - } + // WatchCh will receive updates on service (de)registrations and check (de)registrations + ws.Add(svcState.WatchCh) - // Return immediately if the hash is different or the Watch returns true (indicating timeout fired). - if lastHash != hash || ws.Watch(timeout.C) { - break - } + reply := c.Agent.ServiceHTTPBasedChecks(reqReal.ServiceID) - // Watch returned false indicating a change was detected, loop and repeat - // the call to ServiceHTTPBasedChecks to load the new value. - // If agent sync is paused it means local state is being bulk-edited e.g. config reload. - if syncPauseCh := c.Agent.SyncPausedCh(); syncPauseCh != nil { - // Wait for pause to end or for the timeout to elapse. - select { - case <-syncPauseCh: - case <-timeout.C: - break WATCH_LOOP + hash, err := hashChecks(reply) + if err != nil { + return "", result, fmt.Errorf("Internal cache failure: %v", err) } - } - } - result.Value = &resp + return hash, reply, nil + }, + ) + + result.Value = resp // Below is a purely synthetic index to keep the caching happy. if opts.LastResult == nil { diff --git a/agent/cache-types/service_checks_test.go b/agent/cache-types/service_checks_test.go index 5b204d55f5d5..b8af94794820 100644 --- a/agent/cache-types/service_checks_test.go +++ b/agent/cache-types/service_checks_test.go @@ -1,13 +1,14 @@ -package cachetype_test +package cachetype import ( + "fmt" "github.com/hashicorp/consul/agent/cache" - cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/checks" "github.com/hashicorp/consul/agent/local" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/token" "github.com/hashicorp/consul/types" + "github.com/hashicorp/go-memdb" "github.com/stretchr/testify/require" "testing" "time" @@ -41,7 +42,7 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { // Create mockAgent and cache type a := newMockAgent() a.LocalState().SetServiceState(&svcState) - typ := cachetype.ServiceHTTPChecks{Agent: a} + typ := ServiceHTTPChecks{Agent: a} // Adding TTL check should not yield result from Fetch since TTL checks aren't tracked if err := a.AddCheck(*chkTypes[2]); err != nil { @@ -49,16 +50,16 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { } result, err := typ.Fetch( cache.FetchOptions{}, - &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, + &ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, ) if err != nil { t.Fatalf("failed to fetch: %v", err) } - got, ok := result.Value.(*[]structs.CheckType) + got, ok := result.Value.([]structs.CheckType) if !ok { - t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("fetched value of wrong type, got %T, want []structs.CheckType", result.Value) } - require.Empty(t, *got) + require.Empty(t, got) // Adding HTTP check should yield check in Fetch if err := a.AddCheck(*chkTypes[0]); err != nil { @@ -66,7 +67,7 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { } result, err = typ.Fetch( cache.FetchOptions{}, - &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + &ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, ) if err != nil { t.Fatalf("failed to fetch: %v", err) @@ -74,14 +75,13 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { if result.Index != 1 { t.Fatalf("expected index of 1 after first cache hit, got %d", result.Index) } - - got, ok = result.Value.(*[]structs.CheckType) + got, ok = result.Value.([]structs.CheckType) if !ok { - t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("fetched value of wrong type, got %T, want []structs.CheckType", result.Value) } want := chkTypes[0:1] - for i, c := range *got { - require.Equal(t, c, *want[i]) + for i, c := range want { + require.Equal(t, *c, got[i]) } // Adding GRPC check should yield both checks in Fetch @@ -90,7 +90,7 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { } result2, err := typ.Fetch( cache.FetchOptions{LastResult: &result}, - &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + &ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, ) if err != nil { t.Fatalf("failed to fetch: %v", err) @@ -99,13 +99,13 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { t.Fatalf("expected index of 2 after second request, got %d", result2.Index) } - got, ok = result2.Value.(*[]structs.CheckType) + got, ok = result2.Value.([]structs.CheckType) if !ok { - t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("fetched value of wrong type, got %T, want []structs.CheckType", got) } want = chkTypes[0:2] - for i, c := range *got { - require.Equal(t, c, *want[i]) + for i, c := range want { + require.Equal(t, *c, got[i]) } // Removing GRPC check should yield HTTP check in Fetch @@ -114,7 +114,7 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { } result3, err := typ.Fetch( cache.FetchOptions{LastResult: &result2}, - &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, + &ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID}, ) if err != nil { t.Fatalf("failed to fetch: %v", err) @@ -123,19 +123,19 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { t.Fatalf("expected index of 3 after third request, got %d", result3.Index) } - got, ok = result3.Value.(*[]structs.CheckType) + got, ok = result3.Value.([]structs.CheckType) if !ok { - t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("fetched value of wrong type, got %T, want []structs.CheckType", got) } want = chkTypes[0:1] - for i, c := range *got { - require.Equal(t, c, *want[i]) + for i, c := range want { + require.Equal(t, *c, got[i]) } // Fetching again should yield no change in result nor index result4, err := typ.Fetch( cache.FetchOptions{LastResult: &result3}, - &cachetype.ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, + &ServiceHTTPChecksRequest{ServiceID: svcState.Service.ID, MaxQueryTime: 100 * time.Millisecond}, ) if err != nil { t.Fatalf("failed to fetch: %v", err) @@ -144,19 +144,19 @@ func TestServiceHTTPChecks_Fetch(t *testing.T) { t.Fatalf("expected index of 3 after fetch timeout, got %d", result4.Index) } - got, ok = result4.Value.(*[]structs.CheckType) + got, ok = result4.Value.([]structs.CheckType) if !ok { - t.Fatalf("fetched value of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("fetched value of wrong type, got %T, want []structs.CheckType", got) } want = chkTypes[0:1] - for i, c := range *got { - require.Equal(t, c, *want[i]) + for i, c := range want { + require.Equal(t, *c, got[i]) } } func TestServiceHTTPChecks_badReqType(t *testing.T) { a := newMockAgent() - typ := cachetype.ServiceHTTPChecks{Agent: a} + typ := ServiceHTTPChecks{Agent: a} // Fetch _, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest( @@ -166,16 +166,14 @@ func TestServiceHTTPChecks_badReqType(t *testing.T) { } type mockAgent struct { - state *local.State - pauseCh <-chan struct{} - checks []structs.CheckType + state *local.State + checks []structs.CheckType } func newMockAgent() *mockAgent { m := mockAgent{ - state: local.NewState(local.Config{NodeID: "host"}, nil, new(token.Store)), - pauseCh: make(chan struct{}), - checks: make([]structs.CheckType, 0), + state: local.NewState(local.Config{NodeID: "host"}, nil, new(token.Store)), + checks: make([]structs.CheckType, 0), } m.state.TriggerSyncChanges = func() {} return &m @@ -189,8 +187,14 @@ func (m *mockAgent) LocalState() *local.State { return m.state } -func (m *mockAgent) SyncPausedCh() <-chan struct{} { - return m.pauseCh +func (m *mockAgent) LocalBlockingQuery(alwaysBlock bool, hash string, wait time.Duration, + fn func(ws memdb.WatchSet) (string, interface{}, error)) (string, interface{}, error) { + + hash, err := hashChecks(m.checks) + if err != nil { + return "", nil, fmt.Errorf("failed to hash checks: %+v", m.checks) + } + return hash, m.checks, nil } func (m *mockAgent) AddCheck(check structs.CheckType) error { diff --git a/agent/event_endpoint.go b/agent/event_endpoint.go index 08516e35ee37..f0acff192014 100644 --- a/agent/event_endpoint.go +++ b/agent/event_endpoint.go @@ -13,11 +13,6 @@ import ( "github.com/hashicorp/consul/agent/structs" ) -const ( - // maxQueryTime is used to bound the limit of a blocking query - maxQueryTime = 600 * time.Second -) - // EventFire is used to fire a new event func (s *HTTPServer) EventFire(resp http.ResponseWriter, req *http.Request) (interface{}, error) { diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 608c4d20915d..05bde60d3ea8 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -552,12 +552,12 @@ func (s *state) handleUpdateConnectProxy(u cache.UpdateEvent, snap *ConfigSnapsh snap.ConnectProxy.UpstreamEndpoints[pq] = resp.Nodes case strings.HasPrefix(u.CorrelationID, svcChecksWatchIDPrefix): - resp, ok := u.Result.(*[]structs.CheckType) + resp, ok := u.Result.([]structs.CheckType) if !ok { - return fmt.Errorf("invalid type for service checks response: %T, want: *[]structs.CheckType", u.Result) + return fmt.Errorf("invalid type for service checks response: %T, want: []structs.CheckType", u.Result) } svcID := strings.TrimPrefix(u.CorrelationID, svcChecksWatchIDPrefix) - snap.ConnectProxy.WatchedServiceChecks[svcID] = *resp + snap.ConnectProxy.WatchedServiceChecks[svcID] = resp default: return errors.New("unknown correlation ID") diff --git a/agent/service_checks_test.go b/agent/service_checks_test.go index bdad4b978f4c..079e26c9105c 100644 --- a/agent/service_checks_test.go +++ b/agent/service_checks_test.go @@ -56,8 +56,7 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { TTL: 10 * time.Second, }, } - - // Adding first TTL type should lead to a timeout, since only HTTP-based checks are watched + // Adding TTL type should lead to a timeout, since only HTTP-based checks are watched if err := a.AddService(&service, chkTypes[2:], false, "", ConfigSourceLocal); err != nil { t.Fatalf("failed to add service: %v", err) } @@ -80,13 +79,13 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { t.Fatal("didn't get cache update event") } - got, ok := val.Result.(*[]structs.CheckType) + got, ok := val.Result.([]structs.CheckType) if !ok { - t.Fatalf("notified of result of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) } want := chkTypes[0:2] - for i, c := range *got { - require.Equal(t, c, *want[i]) + for i, c := range want { + require.Equal(t, *c, got[i]) } // Removing the GRPC check should leave only the HTTP check @@ -100,12 +99,12 @@ func TestAgent_ServiceHTTPChecksNotification(t *testing.T) { t.Fatal("didn't get cache update event") } - got, ok = val.Result.(*[]structs.CheckType) + got, ok = val.Result.([]structs.CheckType) if !ok { - t.Fatalf("notified of result of wrong type, got %T, want *[]structs.CheckType", got) + t.Fatalf("notified of result of wrong type, got %T, want []structs.CheckType", got) } want = chkTypes[0:1] - for i, c := range *got { - require.Equal(t, c, *want[i]) + for i, c := range want { + require.Equal(t, *c, got[i]) } } From 1eb3452a27708d78fc2d00c579b18908ea02cf5d Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 17 Sep 2019 17:16:10 -0600 Subject: [PATCH 59/80] Fix tests to account for tls validation --- agent/config/runtime_test.go | 5 ++++- agent/structs/testing_catalog.go | 2 ++ api/agent_test.go | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index a6109d8c2a9d..82a0b0e8c5fc 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -3976,7 +3976,8 @@ func TestFullConfig(t *testing.T) { "path": "/health", "local_path_port": 8080, "listener_port": 21500, - "protocol": "http" + "protocol": "http", + "tls_skip_verify": true } ] }, @@ -4591,6 +4592,7 @@ func TestFullConfig(t *testing.T) { local_path_port = 8080 listener_port = 21500 protocol = "http" + tls_skip_verify = true } ] } @@ -5176,6 +5178,7 @@ func TestFullConfig(t *testing.T) { LocalPathPort: 8080, ListenerPort: 21500, Protocol: "http", + TLSSkipVerify: true, }, }, }, diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index f25c4ef5bd07..8c1a59bd5284 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -64,11 +64,13 @@ func TestNodeServiceExpose(t testing.T) *NodeService { Path: "/foo", LocalPathPort: 8080, ListenerPort: 21500, + TLSSkipVerify: true, }, { Path: "/bar", LocalPathPort: 8080, ListenerPort: 21501, + TLSSkipVerify: true, }, }, }, diff --git a/api/agent_test.go b/api/agent_test.go index 3b8e61f4a7b0..4a066e667666 100644 --- a/api/agent_test.go +++ b/api/agent_test.go @@ -1580,6 +1580,7 @@ func TestAgentService_ExposeChecks(t *testing.T) { ListenerPort: 21500, Path: "/metrics", Protocol: "http2", + TLSSkipVerify: true, } reg := AgentServiceRegistration{ Kind: ServiceKindConnectProxy, From 671cdaf3c579f0ed043704f90e0b50f9e380d556 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 17 Sep 2019 17:40:07 -0600 Subject: [PATCH 60/80] Fix panic in addCheck --- agent/agent.go | 10 ++++++---- agent/agent_endpoint_test.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 25404716d216..e4e62e37d896 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2394,10 +2394,12 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, // Get the address of the proxy for this service if it exists // Need its config to know whether we should reroute checks to it var proxy *structs.NodeService - for _, svc := range a.State.Services() { - if svc.Proxy.DestinationServiceID == service.ID { - proxy = svc - break + if service != nil { + for _, svc := range a.State.Services() { + if svc.Proxy.DestinationServiceID == service.ID { + proxy = svc + break + } } } diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index 7136b45bd782..f7217bcdc229 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -1896,7 +1896,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { t.Parallel() a := NewTestAgent(t, t.Name(), TestACLConfigNew()) defer a.Shutdown() - testrpc.WaitForLeader(t, a.RPC, "dc1") + testrpc.WaitForTestAgent(t, a.RPC, "dc1") nodeCheck := &structs.CheckDefinition{ Name: "test", From f2982b075114fe24c3e6fe9015bd5d490d3e2741 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 18 Sep 2019 10:52:31 -0600 Subject: [PATCH 61/80] Restrict exposed check sources to localhost and agent addr --- agent/agent.go | 8 +++++ agent/structs/connect_proxy_config.go | 3 ++ agent/xds/listeners.go | 46 +++++++++++++++++++++------ agent/xds/server.go | 7 ++++ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index e4e62e37d896..a1bb17d0d435 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -679,6 +679,7 @@ func (a *Agent) listenAndServeGRPC() error { Authz: a, ResolveToken: a.resolveToken, CheckFetcher: a, + CfgFetcher: a, } a.xdsServer.Initialize() @@ -2705,6 +2706,9 @@ func (a *Agent) removeCheckLocked(checkID types.CheckID, persist bool) error { } func (a *Agent) ServiceHTTPBasedChecks(serviceID string) []structs.CheckType { + a.stateLock.Lock() + defer a.stateLock.Unlock() + var chkTypes = make([]structs.CheckType, 0) for _, c := range a.checkHTTPs { if c.ServiceID == serviceID { @@ -2719,6 +2723,10 @@ func (a *Agent) ServiceHTTPBasedChecks(serviceID string) []structs.CheckType { return chkTypes } +func (a *Agent) AdvertiseAddrLAN() string { + return a.config.AdvertiseAddrLAN.String() +} + // resolveProxyCheckAddress returns the best address to use for a TCP check of // the proxy's public listener. It expects the input to already have default // values populated by applyProxyConfigDefaults. It may return an empty string diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index fab91d54d1ff..ca28c63c67cf 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -357,6 +357,9 @@ type ExposePath struct { // CACert contains the PEM encoded CA file read from CAFile CACert string + + // ParsedFromCheck is set if this path was parsed from a registered check + ParsedFromCheck bool } func (e *ExposeConfig) ToAPI() api.ExposeConfig { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index e62c8fcb896b..601e8d9f065b 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -98,7 +98,7 @@ func (s *Server) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnaps clusterName = makeExposeClusterName(path.LocalPathPort) } - l, err := s.makeExposedCheckListener(cfgSnap, clusterName, path.Path, path.Protocol, path.ListenerPort) + l, err := s.makeExposedCheckListener(cfgSnap, clusterName, path) if err != nil { return nil, err } @@ -172,6 +172,9 @@ func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { return path, fmt.Errorf("failed to parse port from '%s': %v", check.ProxyGRPC, err) } } + + path.ParsedFromCheck = true + return path, nil } @@ -372,7 +375,7 @@ func (s *Server) makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token stri return l, err } -func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, cluster, path, protocol string, port int) (proto.Message, error) { +func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, cluster string, path structs.ExposePath) (proto.Message, error) { cfg, err := ParseProxyConfig(cfgSnap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns @@ -392,23 +395,46 @@ func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, clus // Strip any special characters from path to make a valid and hopefully unique name r := regexp.MustCompile(`[^a-zA-Z0-9]+`) - strippedPath := r.ReplaceAllString(path, "") + strippedPath := r.ReplaceAllString(path.Path, "") listenerName := fmt.Sprintf("exposed_path_%s", strippedPath) - l := makeListener(listenerName, addr, port) + l := makeListener(listenerName, addr, path.ListenerPort) - filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, port) + filterName := fmt.Sprintf("exposed_path_filter_%s_%d", strippedPath, path.ListenerPort) - f, err := makeListenerFilter(false, protocol, filterName, cluster, "", path, true) + f, err := makeListenerFilter(false, path.Protocol, filterName, cluster, "", path.Path, true) if err != nil { return nil, err } - l.FilterChains = []envoylistener.FilterChain{ - { - Filters: []envoylistener.Filter{f}, - }, + chain := envoylistener.FilterChain{ + Filters: []envoylistener.Filter{f}, } + + // For registered checks restrict traffic sources to localhost and Consul's advertise addr + if path.ParsedFromCheck { + + // For the advertise addr we use a CidrRange that only matches one address + advertise := s.CfgFetcher.AdvertiseAddrLAN() + + // Get prefix length based on whether address is ipv4 (32 bits) or ipv6 (128 bits) + advertiseLen := 32 + ip := net.ParseIP(advertise) + if ip != nil && strings.Contains(advertise, ":") { + advertiseLen = 128 + } + + chain.FilterChainMatch = &envoylistener.FilterChainMatch{ + SourcePrefixRanges: []*envoycore.CidrRange{ + {AddressPrefix: "127.0.0.1", PrefixLen: &types.UInt32Value{Value: 8}}, + {AddressPrefix: "::1", PrefixLen: &types.UInt32Value{Value: 128}}, + {AddressPrefix: advertise, PrefixLen: &types.UInt32Value{Value: uint32(advertiseLen)}}, + }, + } + } + + l.FilterChains = []envoylistener.FilterChain{chain} + return l, err } diff --git a/agent/xds/server.go b/agent/xds/server.go index 33d22621c57e..0e6ae8ea832c 100644 --- a/agent/xds/server.go +++ b/agent/xds/server.go @@ -103,6 +103,12 @@ type HTTPCheckFetcher interface { ServiceHTTPBasedChecks(serviceID string) []structs.CheckType } +// ConfigFetcher is the interface the agent needs to expose +// for the xDS server to fetch agent config, currently only one field is fetched +type ConfigFetcher interface { + AdvertiseAddrLAN() string +} + // ConfigManager is the interface xds.Server requires to consume proxy config // updates. It's satisfied normally by the agent's proxycfg.Manager, but allows // easier testing without several layers of mocked cache, local state and @@ -128,6 +134,7 @@ type Server struct { // there has been no recent DiscoveryRequest). AuthCheckFrequency time.Duration CheckFetcher HTTPCheckFetcher + CfgFetcher ConfigFetcher } // Initialize will finish configuring the Server for first use. From 901253c8890542ce9e15b0fda561de64c8bf12ea Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 18 Sep 2019 12:18:12 -0600 Subject: [PATCH 62/80] Fix some tests --- agent/agent_endpoint_test.go | 50 +++++++++++++++---------- agent/structs/structs_filtering_test.go | 5 +++ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index f7217bcdc229..fd5f101b32e6 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -1896,7 +1896,7 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { t.Parallel() a := NewTestAgent(t, t.Name(), TestACLConfigNew()) defer a.Shutdown() - testrpc.WaitForTestAgent(t, a.RPC, "dc1") + testrpc.WaitForLeader(t, a.RPC, "dc1") nodeCheck := &structs.CheckDefinition{ Name: "test", @@ -1980,39 +1980,51 @@ func TestAgent_RegisterCheck_ACLDeny(t *testing.T) { require.NotNil(t, nodeToken) t.Run("no token - node check", func(t *testing.T) { - req, _ := http.NewRequest("PUT", "/v1/agent/check/register", jsonReader(nodeCheck)) - _, err := a.srv.AgentRegisterCheck(nil, req) - require.True(t, acl.IsErrPermissionDenied(err)) + retry.Run(t, func(r *retry.R) { + req, _ := http.NewRequest("PUT", "/v1/agent/check/register", jsonReader(nodeCheck)) + _, err := a.srv.AgentRegisterCheck(nil, req) + require.True(r, acl.IsErrPermissionDenied(err)) + }) }) t.Run("svc token - node check", func(t *testing.T) { - req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+svcToken.SecretID, jsonReader(nodeCheck)) - _, err := a.srv.AgentRegisterCheck(nil, req) - require.True(t, acl.IsErrPermissionDenied(err)) + retry.Run(t, func(r *retry.R) { + req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+svcToken.SecretID, jsonReader(nodeCheck)) + _, err := a.srv.AgentRegisterCheck(nil, req) + require.True(r, acl.IsErrPermissionDenied(err)) + }) }) t.Run("node token - node check", func(t *testing.T) { - req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+nodeToken.SecretID, jsonReader(nodeCheck)) - _, err := a.srv.AgentRegisterCheck(nil, req) - require.NoError(t, err) + retry.Run(t, func(r *retry.R) { + req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+nodeToken.SecretID, jsonReader(nodeCheck)) + _, err := a.srv.AgentRegisterCheck(nil, req) + require.NoError(r, err) + }) }) t.Run("no token - svc check", func(t *testing.T) { - req, _ := http.NewRequest("PUT", "/v1/agent/check/register", jsonReader(svcCheck)) - _, err := a.srv.AgentRegisterCheck(nil, req) - require.True(t, acl.IsErrPermissionDenied(err)) + retry.Run(t, func(r *retry.R) { + req, _ := http.NewRequest("PUT", "/v1/agent/check/register", jsonReader(svcCheck)) + _, err := a.srv.AgentRegisterCheck(nil, req) + require.True(r, acl.IsErrPermissionDenied(err)) + }) }) t.Run("node token - svc check", func(t *testing.T) { - req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+nodeToken.SecretID, jsonReader(svcCheck)) - _, err := a.srv.AgentRegisterCheck(nil, req) - require.True(t, acl.IsErrPermissionDenied(err)) + retry.Run(t, func(r *retry.R) { + req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+nodeToken.SecretID, jsonReader(svcCheck)) + _, err := a.srv.AgentRegisterCheck(nil, req) + require.True(r, acl.IsErrPermissionDenied(err)) + }) }) t.Run("svc token - svc check", func(t *testing.T) { - req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+svcToken.SecretID, jsonReader(svcCheck)) - _, err := a.srv.AgentRegisterCheck(nil, req) - require.NoError(t, err) + retry.Run(t, func(r *retry.R) { + req, _ := http.NewRequest("PUT", "/v1/agent/check/register?token="+svcToken.SecretID, jsonReader(svcCheck)) + _, err := a.srv.AgentRegisterCheck(nil, req) + require.NoError(r, err) + }) }) } diff --git a/agent/structs/structs_filtering_test.go b/agent/structs/structs_filtering_test.go index 6b3ae2091387..ba1ef5865fbf 100644 --- a/agent/structs/structs_filtering_test.go +++ b/agent/structs/structs_filtering_test.go @@ -109,6 +109,11 @@ var expectedFieldConfigPaths bexpr.FieldConfigurations = bexpr.FieldConfiguratio CoerceFn: bexpr.CoerceString, SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, }, + "ParsedFromCheck": &bexpr.FieldConfiguration{ + StructFieldName: "ParsedFromCheck", + CoerceFn: bexpr.CoerceBool, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, } var expectedFieldConfigUpstreams bexpr.FieldConfigurations = bexpr.FieldConfigurations{ From b514c6fa23f198305118b3358a83792a15ef5829 Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 20 Sep 2019 09:30:04 -0600 Subject: [PATCH 63/80] Address config entries PR comments --- agent/agent.go | 17 ++-- agent/config/config.go | 6 ++ agent/service_manager.go | 4 + agent/structs/config_entry.go | 17 +++- agent/structs/connect_proxy_config.go | 20 ++--- agent/structs/structs_filtering_test.go | 9 +- api/config_entry.go | 7 +- api/config_entry_test.go | 86 +++++++++++++++++++ command/config/write/config_write_test.go | 100 ++++++++++++++++++++++ 9 files changed, 237 insertions(+), 29 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index a1bb17d0d435..c72017c58ba8 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -87,6 +87,11 @@ const ( defaultQueryTime = 300 * time.Second ) +var ( + httpAddrRE = regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) + grpcAddrRE = regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") +) + type configSource int const ( @@ -3753,28 +3758,24 @@ func listenerPortKey(svcID, checkID string) string { // grpcInjectAddr injects an ip and port into an address of the form: ip:port[/service] func grpcInjectAddr(existing string, ip string, port int) string { - r := regexp.MustCompile("(.*)((?::)(?:[0-9]+))(.*)$") - portRepl := fmt.Sprintf("${1}:%d${3}", port) - out := r.ReplaceAllString(existing, portRepl) + out := grpcAddrRE.ReplaceAllString(existing, portRepl) addrRepl := fmt.Sprintf("%s${2}${3}", ip) - out = r.ReplaceAllString(out, addrRepl) + out = grpcAddrRE.ReplaceAllString(out, addrRepl) return out } // httpInjectAddr injects a port then an IP into a URL func httpInjectAddr(url string, ip string, port int) string { - r := regexp.MustCompile(`^(http[s]?://)(\[.*?\]|\[?[\w\-\.]+)(:\d+)?([^?]*)(\?.*)?$`) - portRepl := fmt.Sprintf("${1}${2}:%d${4}${5}", port) - out := r.ReplaceAllString(url, portRepl) + out := httpAddrRE.ReplaceAllString(url, portRepl) // Ensure that ipv6 addr is enclosed in brackets (RFC 3986) ip = fixIPv6(ip) addrRepl := fmt.Sprintf("${1}%s${3}${4}${5}", ip) - out = r.ReplaceAllString(out, addrRepl) + out = httpAddrRE.ReplaceAllString(out, addrRepl) return out } diff --git a/agent/config/config.go b/agent/config/config.go index db957106f0e0..449bba77b445 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -554,6 +554,12 @@ type ExposePath struct { // CAFile is the path to the PEM encoded CA cert used to verify client certificates. CAFile *string `json:"ca_file,omitempty" hcl:"ca_file" mapstructure:"ca_file"` + + // CertFile is the path to the PEM encoded CA cert used to verify client certificates. + CertFile *string `json:"cert_file,omitempty" hcl:"cert_file" mapstructure:"cert_file"` + + // KeyFile is the path to the PEM encoded CA cert used to verify client certificates. + KeyFile *string `json:"key_file,omitempty" hcl:"key_file" mapstructure:"key_file"` } // AutoEncrypt is the agent-global auto_encrypt configuration. diff --git a/agent/service_manager.go b/agent/service_manager.go index 001d125da67d..07eff37dfbb1 100644 --- a/agent/service_manager.go +++ b/agent/service_manager.go @@ -363,6 +363,10 @@ func (s *serviceConfigWatch) mergeServiceConfig() (*structs.NodeService, error) return nil, err } + if err := mergo.Merge(&ns.Proxy.Expose, s.defaults.Expose); err != nil { + return nil, err + } + if ns.Proxy.MeshGateway.Mode == structs.MeshGatewayModeDefault { ns.Proxy.MeshGateway.Mode = s.defaults.MeshGateway.Mode } diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 839ae8ee6a3b..8bb834c4fa8e 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -299,10 +299,19 @@ func DecodeConfigEntry(raw map[string]interface{}) (ConfigEntry, error) { func ConfigEntryDecodeRulesForKind(kind string) (skipWhenPatching []string, translateKeysDict map[string]string, err error) { switch kind { case ProxyDefaults: - return nil, map[string]string{ - "mesh_gateway": "meshgateway", - "config": "", - }, nil + return []string{ + "expose.paths", + "Expose.Paths", + }, map[string]string{ + "local_path_port": "localpathport", + "listener_port": "listenerport", + "key_file": "keyfile", + "cert_file": "certfile", + "ca_file": "cafile", + "tls_skip_verify": "tlsskipverify", + "mesh_gateway": "meshgateway", + "config": "", + }, nil case ServiceDefaults: return nil, map[string]string{ "mesh_gateway": "meshgateway", diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index ca28c63c67cf..7169bc306446 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" - "io/ioutil" "log" ) @@ -355,8 +354,11 @@ type ExposePath struct { // CAFile is the path to the PEM encoded CA cert used to verify client certificates. CAFile string `json:",omitempty"` - // CACert contains the PEM encoded CA file read from CAFile - CACert string + // CertFile is the path fo the PEM encoded client certificate to authenticate Envoy + CertFile string `json:",omitempty"` + + // KeyFile is the path fo the PEM encoded client certificate to authenticate Envoy + KeyFile string `json:",omitempty"` // ParsedFromCheck is set if this path was parsed from a registered check ParsedFromCheck bool @@ -385,7 +387,8 @@ func (p *ExposePath) ToAPI() api.ExposePath { Protocol: p.Protocol, TLSSkipVerify: p.TLSSkipVerify, CAFile: p.CAFile, - CACert: p.CACert, + CertFile: p.CertFile, + KeyFile: p.KeyFile, } } @@ -397,14 +400,5 @@ func (e *ExposeConfig) Finalize(l *log.Logger) { if path.Protocol == "" { path.Protocol = defaultExposeProtocol } - - if path.CAFile != "" { - b, err := ioutil.ReadFile(path.CAFile) - if err != nil { - l.Printf("[WARN] envoy: failed to read CAFile '%s': %v", path.CAFile, err) - continue - } - path.CACert = string(b) - } } } diff --git a/agent/structs/structs_filtering_test.go b/agent/structs/structs_filtering_test.go index ba1ef5865fbf..2cbf73aa57da 100644 --- a/agent/structs/structs_filtering_test.go +++ b/agent/structs/structs_filtering_test.go @@ -104,8 +104,13 @@ var expectedFieldConfigPaths bexpr.FieldConfigurations = bexpr.FieldConfiguratio CoerceFn: bexpr.CoerceString, SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, }, - "CACert": &bexpr.FieldConfiguration{ - StructFieldName: "CACert", + "CertFile": &bexpr.FieldConfiguration{ + StructFieldName: "CertFile", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, + }, + "KeyFile": &bexpr.FieldConfiguration{ + StructFieldName: "KeyFile", CoerceFn: bexpr.CoerceString, SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, }, diff --git a/api/config_entry.go b/api/config_entry.go index 59e9b5b41d14..ebd70bcc6bca 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -87,8 +87,11 @@ type ExposePath struct { // CAFile is the path to the PEM encoded CA cert used to verify client certificates. CAFile string `json:",omitempty"` - // CACert contains the PEM encoded CA file read from CAFile - CACert string `json:",omitempty"` + // CertFile is the path fo the PEM encoded client certificate to authenticate Envoy + CertFile string `json:",omitempty"` + + // KeyFile is the path fo the PEM encoded client certificate to authenticate Envoy + KeyFile string `json:",omitempty"` } type ServiceConfigEntry struct { diff --git a/api/config_entry_test.go b/api/config_entry_test.go index 13a6d6ee1df9..3d32ea7300af 100644 --- a/api/config_entry_test.go +++ b/api/config_entry_test.go @@ -195,6 +195,92 @@ func TestDecodeConfigEntry(t *testing.T) { expect ConfigEntry expectErr string }{ + { + name: "expose-paths: kitchen sink proxy", + body: ` + { + "Kind": "proxy-defaults", + "Name": "global", + "Expose": { + "Checks": true, + "Paths": [ + { + "LocalPathPort": 8080, + "ListenerPort": 21500, + "Path": "/healthz", + "Protocol": "http2", + "TLSSkipVerify": true, + "CAFile": "ca.pem", + "CertFile": "cert.pem", + "KeyFile": "key.pem" + } + ] + } + } + `, + expect: &ProxyConfigEntry{ + Kind: "proxy-defaults", + Name: "global", + Expose: ExposeConfig{ + Checks: true, + Paths: []ExposePath{ + { + LocalPathPort: 8080, + ListenerPort: 21500, + Path: "/healthz", + Protocol: "http2", + TLSSkipVerify: true, + CAFile: "ca.pem", + CertFile: "cert.pem", + KeyFile: "key.pem", + }, + }, + }, + }, + }, + { + name: "expose-paths: kitchen sink service default", + body: ` + { + "Kind": "service-defaults", + "Name": "global", + "Expose": { + "Checks": true, + "Paths": [ + { + "LocalPathPort": 8080, + "ListenerPort": 21500, + "Path": "/healthz", + "Protocol": "http2", + "TLSSkipVerify": true, + "CAFile": "ca.pem", + "CertFile": "cert.pem", + "KeyFile": "key.pem" + } + ] + } + } + `, + expect: &ServiceConfigEntry{ + Kind: "service-defaults", + Name: "global", + Expose: ExposeConfig{ + Checks: true, + Paths: []ExposePath{ + { + LocalPathPort: 8080, + ListenerPort: 21500, + Path: "/healthz", + Protocol: "http2", + TLSSkipVerify: true, + CAFile: "ca.pem", + CertFile: "cert.pem", + KeyFile: "key.pem", + }, + }, + }, + }, + }, { name: "proxy-defaults", body: ` diff --git a/command/config/write/config_write_test.go b/command/config/write/config_write_test.go index 22ac7c7d550a..e31dcc14df18 100644 --- a/command/config/write/config_write_test.go +++ b/command/config/write/config_write_test.go @@ -1042,6 +1042,106 @@ func TestParseConfigEntry(t *testing.T) { Name: "main", }, }, + { + name: "expose paths: kitchen sink proxy defaults", + snake: ` + kind = "proxy-defaults" + name = "global" + expose = { + checks = true + paths = [ + { + local_path_port = 8080 + listener_port = 21500 + path = "/healthz" + protocol = "http2" + tls_skip_verify = true + ca_file = "ca.pem" + cert_file = "cert.pem" + key_file = "key.pem" + } + ] + }`, + camel: ` + Kind = "proxy-defaults" + Name = "global" + Expose = { + Checks = true + Paths = [ + { + LocalPathPort = 8080 + ListenerPort = 21500 + Path = "/healthz" + Protocol = "http2" + TLSSkipVerify = true + CAFile = "ca.pem" + CertFile = "cert.pem" + KeyFile = "key.pem" + } + ] + }`, + snakeJSON: ` + { + "kind": "proxy-defaults", + "name": "global", + "expose": { + "checks": true, + "paths": [ + { + "local_path_port": 8080, + "listener_port": 21500, + "path": "/healthz", + "protocol": "http2", + "tls_skip_verify": true, + "ca_file": "ca.pem", + "cert_file": "cert.pem", + "key_file": "key.pem" + } + ] + } + } + `, + camelJSON: ` + { + "Kind": "proxy-defaults", + "Name": "global", + "Expose": { + "Checks": true, + "Paths": [ + { + "LocalPathPort": 8080, + "ListenerPort": 21500, + "Path": "/healthz", + "Protocol": "http2", + "TLSSkipVerify": true, + "CAFile": "ca.pem", + "CertFile": "cert.pem", + "KeyFile": "key.pem" + } + ] + } + } + `, + expect: &api.ProxyConfigEntry{ + Kind: "proxy-defaults", + Name: "global", + Expose: api.ExposeConfig{ + Checks: true, + Paths: []api.ExposePath{ + { + ListenerPort: 21500, + Path: "/healthz", + LocalPathPort: 8080, + Protocol: "http2", + TLSSkipVerify: true, + CAFile: "ca.pem", + CertFile: "cert.pem", + KeyFile: "key.pem", + }, + }, + }, + }, + }, } { tc := tc From c8366b24cba3afb43e2cc6656ed74cb1f4b7d241 Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 20 Sep 2019 14:23:31 -0600 Subject: [PATCH 64/80] Fix filtering test --- agent/structs/connect_proxy_config.go | 17 +++++++++-------- api/config_entry.go | 3 +++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 7169bc306446..9e32ba7808d7 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -381,14 +381,15 @@ func (e *ExposeConfig) ToAPI() api.ExposeConfig { func (p *ExposePath) ToAPI() api.ExposePath { return api.ExposePath{ - ListenerPort: p.ListenerPort, - Path: p.Path, - LocalPathPort: p.LocalPathPort, - Protocol: p.Protocol, - TLSSkipVerify: p.TLSSkipVerify, - CAFile: p.CAFile, - CertFile: p.CertFile, - KeyFile: p.KeyFile, + ListenerPort: p.ListenerPort, + Path: p.Path, + LocalPathPort: p.LocalPathPort, + Protocol: p.Protocol, + TLSSkipVerify: p.TLSSkipVerify, + CAFile: p.CAFile, + CertFile: p.CertFile, + KeyFile: p.KeyFile, + ParsedFromCheck: p.ParsedFromCheck, } } diff --git a/api/config_entry.go b/api/config_entry.go index ebd70bcc6bca..31d51fd872c6 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -92,6 +92,9 @@ type ExposePath struct { // KeyFile is the path fo the PEM encoded client certificate to authenticate Envoy KeyFile string `json:",omitempty"` + + // ParsedFromCheck is set if this path was parsed from a registered check + ParsedFromCheck bool } type ServiceConfigEntry struct { From 91b9855dbc5ec49db365421639769f9c052af463 Mon Sep 17 00:00:00 2001 From: freddygv Date: Fri, 20 Sep 2019 14:23:58 -0600 Subject: [PATCH 65/80] Add documentation --- .../config-entries/proxy-defaults.html.md | 22 ++++++ .../config-entries/service-defaults.html.md | 22 ++++++ website/source/docs/agent/options.html.md | 10 +++ website/source/docs/agent/services.html.md | 13 ++++ website/source/docs/connect/proxies/envoy.md | 1 + .../registration/service-registration.html.md | 71 +++++++++++++++++++ 6 files changed, 139 insertions(+) diff --git a/website/source/docs/agent/config-entries/proxy-defaults.html.md b/website/source/docs/agent/config-entries/proxy-defaults.html.md index bf9b40c10e0e..5df8f334ec12 100644 --- a/website/source/docs/agent/config-entries/proxy-defaults.html.md +++ b/website/source/docs/agent/config-entries/proxy-defaults.html.md @@ -54,6 +54,28 @@ config { for all proxies. Added in v1.6.0. - `Mode` `(string: "")` - One of `none`, `local`, or `remote`. + +- `Expose` `(ExposeConfig: )` - Controls the default + [expose path configuration](/docs/connect/registration/service-registration.html#expose-paths-configuration-reference) + for Envoy. Added in v1.6.2. + + Exposing paths through Envoy enables a service to protect itself by only listening on localhost, while still allowing + non-Connect-enabled applications to contact an HTTP endpoint. + Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. + + - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's + [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from + [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). + This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running + Consul on Kubernetes, and Consul agents run in their own pods. + - `Paths` `array: []` - A list of paths to expose through Envoy. + - `Path` `(string: "")` - The HTTP path to expose. + - `LocalPathPort` `(int: 0)` - The port where the local service is listening for connections to the path. + - `ListenerPort` `(int: 0)` - The port where the proxy will listen for connections. This port must be available + for the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, + but the proxy registration will not fail. + - `Protocol` `(string: "http")` - Sets the protocol of the listener. One of `http` or `http2`. For gRPC use `http2`. ## ACLs diff --git a/website/source/docs/agent/config-entries/service-defaults.html.md b/website/source/docs/agent/config-entries/service-defaults.html.md index 34b782107a33..daa9cadfe3ae 100644 --- a/website/source/docs/agent/config-entries/service-defaults.html.md +++ b/website/source/docs/agent/config-entries/service-defaults.html.md @@ -43,6 +43,28 @@ Protocol = "http" the TLS [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) value to be changed to a non-connect value when federating with an external system. Added in v1.6.0. + +- `Expose` `(ExposeConfig: )` - Controls the default + [expose path configuration](/docs/connect/registration/service-registration.html#expose-paths-configuration-reference) + for Envoy. Added in v1.6.2. + + Exposing paths through Envoy enables a service to protect itself by only listening on localhost, while still allowing + non-Connect-enabled applications to contact an HTTP endpoint. + Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. + + - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's + [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from + [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). + This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running + Consul on Kubernetes, and Consul agents run in their own pods. + - `Paths` `array: []` - A list of paths to expose through Envoy. + - `Path` `(string: "")` - The HTTP path to expose. + - `LocalPathPort` `(int: 0)` - The port where the local service is listening for connections to the path. + - `ListenerPort` `(int: 0)` - The port where the proxy will listen for connections. This port must be available for + the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, + but the proxy registration will not fail. + - `Protocol` `(string: "http")` - Sets the protocol of the listener. One of `http` or `http2`. For gRPC use `http2`. ## ACLs diff --git a/website/source/docs/agent/options.html.md b/website/source/docs/agent/options.html.md index fb4da1f1f41b..caeee1cd0fd7 100644 --- a/website/source/docs/agent/options.html.md +++ b/website/source/docs/agent/options.html.md @@ -1381,6 +1381,16 @@ default will automatically work with some tooling. number to use for automatically assigned [sidecar service registrations](/docs/connect/registration/sidecar-service.html). Default 21255. Set to `0` to disable automatic port assignment. + * `expose_min_port` - Inclusive minimum port + number to use for automatically assigned + [exposed check listeners](/docs/connect/registration/service-registration.html#expose-paths-configuration-reference). + Default 21500. Set to `0` to disable automatic port assignment. + * `expose_max_port` - Inclusive maximum port + number to use for automatically assigned + [exposed check listeners](/docs/connect/registration/service-registration.html#expose-paths-configuration-reference). + Default 21755. Set to `0` to disable automatic port assignment. * `protocol` Equivalent to the [`-protocol` command-line flag](#_protocol). diff --git a/website/source/docs/agent/services.html.md b/website/source/docs/agent/services.html.md index 7f551752719b..07a2bf56aa0d 100644 --- a/website/source/docs/agent/services.html.md +++ b/website/source/docs/agent/services.html.md @@ -66,6 +66,19 @@ example shows all possible fields, but note that only a few are required. "upstreams": [], "mesh_gateway": { "mode": "local" + }, + "expose": { + "checks": true, + "paths": [ + { + "path": "/healthz", + "protocol": "http2", + "tls_skip_verify": false, + "key_file": "key.pem", + "cert_file": "cert.pem", + "ca_file": "ca.pem" + } + ] } }, "connect": { diff --git a/website/source/docs/connect/proxies/envoy.md b/website/source/docs/connect/proxies/envoy.md index 5b1bcbbbc098..03ebdcd900ea 100644 --- a/website/source/docs/connect/proxies/envoy.md +++ b/website/source/docs/connect/proxies/envoy.md @@ -71,6 +71,7 @@ The dynamic configuration Consul Connect provides to each Envoy instance include - Service-discovery results for upstreams to enable each sidecar proxy to load-balance outgoing connections. - L7 configuration including timeouts and protocol-specific options. + - Configuration to [expose specific HTTP paths](/docs/connect/registration/service-registration.html#expose-paths-configuration-reference). For more information on the parts of the Envoy proxy runtime configuration that are currently controllable via Consul Connect see [Dynamic diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 994c7a607f00..2207173b2987 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -185,6 +185,9 @@ followed by documentation for each attribute. reference](/docs/connect/configuration.html#envoy-options) * `mesh_gateway` `(object: {})` - Specifies the mesh gateway configuration for this proxy. The format is defined in the [Mesh Gateway Configuration Reference](#mesh-gateway-configuration-reference). +* `expose` `(object: {})` - Specifies the configuration to expose HTTP paths through this proxy. + The format is defined in the [Expose Paths Configuration Reference](#expose-paths-configuration-reference), + and is only compatible with an Envoy proxy. ### Mesh Gateway Configuration Reference @@ -237,3 +240,71 @@ registrations](/docs/agent/services.html#service-definition-parameter-case). 2. Proxy Service's `Proxy` configuration 3. The `service-defaults` configuration for the service. 4. The `global` `proxy-defaults`. + +### Expose Paths Configuration Reference + +The following examples show possible configurations to expose HTTP paths through Envoy. + +Exposing paths through Envoy enables a service to protect itself by only listening on localhost, while still allowing +non-Connect-enabled applications to contact an HTTP endpoint. +Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. + +Note that `snake_case` is used here as it works in both [config file and API +registrations](/docs/agent/services.html#service-definition-parameter-case). + +#### Expose listeners in Envoy for HTTP and GRPC checks registered with the local Consul agent + +```json +{ + "expose": { + "checks": true + } +} +``` + +#### Expose an HTTP listener in Envoy at port 2150 that routes to an HTTP server listening at port 8080 + +```json +{ + "expose": { + "paths": [ + { + "path": "/healthz", + "local_path_port": 8080, + "listener_port": 21500 + } + ] + } +} +``` + +#### Expose an HTTP2 listener in Envoy at port 21501 that routes to a gRPC server listening at port 9090 + +```json +{ + "expose": { + "paths": [ + { + "path": "/grpc.health.v1.Health/Check", + "protocol": "http2", + "local_path_port": 9090, + "listener_port": 21500 + } + ] + } +} +``` + +* `checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's + [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from + [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). + This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running + Consul on Kubernetes, and Consul agents run in their own pods. +* `paths` `array: []` - A list of paths to expose through Envoy. + - `path` `(string: "")` - The HTTP path to expose. + - `local_path_port` `(int: 0)` - The port where the local service is listening for connections to the path. + - `listener_port` `(int: 0)` - The port where the proxy will listen for connections. This port must be available for + the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, + but the proxy registration will not fail. + - `protocol` `(string: "http")` - Sets the protocol of the listener. One of `http` or `http2`. For gRPC use `http2`. \ No newline at end of file From a1bf96fbe828a170460e53f385135f7b91754bc9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 23 Sep 2019 16:46:07 -0600 Subject: [PATCH 66/80] Address docs review comments --- .../config-entries/proxy-defaults.html.md | 2 +- .../config-entries/service-defaults.html.md | 2 +- website/source/docs/agent/services.html.md | 41 +++++++++++-------- website/source/docs/connect/proxies/envoy.md | 5 ++- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/website/source/docs/agent/config-entries/proxy-defaults.html.md b/website/source/docs/agent/config-entries/proxy-defaults.html.md index 5df8f334ec12..734398abc50b 100644 --- a/website/source/docs/agent/config-entries/proxy-defaults.html.md +++ b/website/source/docs/agent/config-entries/proxy-defaults.html.md @@ -65,7 +65,7 @@ config { - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's - [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from + [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running Consul on Kubernetes, and Consul agents run in their own pods. diff --git a/website/source/docs/agent/config-entries/service-defaults.html.md b/website/source/docs/agent/config-entries/service-defaults.html.md index daa9cadfe3ae..b3f73ce0b2d2 100644 --- a/website/source/docs/agent/config-entries/service-defaults.html.md +++ b/website/source/docs/agent/config-entries/service-defaults.html.md @@ -54,7 +54,7 @@ Protocol = "http" - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's - [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from + [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running Consul on Kubernetes, and Consul agents run in their own pods. diff --git a/website/source/docs/agent/services.html.md b/website/source/docs/agent/services.html.md index 07a2bf56aa0d..fc24abd9b4ef 100644 --- a/website/source/docs/agent/services.html.md +++ b/website/source/docs/agent/services.html.md @@ -153,6 +153,30 @@ For Consul 0.9.3 and earlier you need to use `enableTagOverride`. Consul 1.0 supports both `enable_tag_override` and `enableTagOverride` but the latter is deprecated and has been removed as of Consul 1.1. +### Checks + +A service can have an associated health check. This is a powerful feature as +it allows a web balancer to gracefully remove failing nodes, a database +to replace a failed secondary, etc. The health check is strongly integrated in +the DNS interface as well. If a service is failing its health check or a +node has any failing system-level check, the DNS interface will omit that +node from any service query. + +There are several check types that have differing required options as +[documented here](/docs/agent/checks.html). The check name is automatically +generated as `service:`. If there are multiple service checks +registered, the ID will be generated as `service::` where +`` is an incrementing number starting from `1`. + +-> **Note:** There is more information about [checks here](/docs/agent/checks.html). + +### Proxy + +Service definitions allow for an optional proxy registration. Proxies used with Connect +are registered as services in Consul's catalog. +See the [Proxy Service Registration](/docs/connect/registration/service-registration.html) reference +for the available configuration options. + ### Connect The `kind` field is used to optionally identify the service as a [Connect @@ -183,23 +207,6 @@ supported "Managed" proxies which are specified with the `connect.proxy` field. [Managed Proxies are deprecated](/docs/connect/proxies/managed-deprecated.html) and the `connect.proxy` field will be removed in a future major release. -### Checks - -A service can have an associated health check. This is a powerful feature as -it allows a web balancer to gracefully remove failing nodes, a database -to replace a failed secondary, etc. The health check is strongly integrated in -the DNS interface as well. If a service is failing its health check or a -node has any failing system-level check, the DNS interface will omit that -node from any service query. - -There are several check types that have differing required options as -[documented here](/docs/agent/checks.html). The check name is automatically -generated as `service:`. If there are multiple service checks -registered, the ID will be generated as `service::` where -`` is an incrementing number starting from `1`. - --> **Note:** There is more information about [checks here](/docs/agent/checks.html). - ### DNS SRV Weights The `weights` field is an optional field to specify the weight of a service in diff --git a/website/source/docs/connect/proxies/envoy.md b/website/source/docs/connect/proxies/envoy.md index 03ebdcd900ea..0a65472831b8 100644 --- a/website/source/docs/connect/proxies/envoy.md +++ b/website/source/docs/connect/proxies/envoy.md @@ -156,7 +156,7 @@ each service such as which protocol they speak. Consul will use this information to configure appropriate proxy settings for that service's proxies and also for the upstream listeners of any downstream service. -Users can define a service's protocol in its [`service-defaults` configuration +One example is how users can define a service's protocol in a [`service-defaults` configuration entry](/docs/agent/config-entries/service-defaults.html). Agents with [`enable_central_service_config`](/docs/agent/options.html#enable_central_service_config) set to true will automatically discover the protocol when configuring a proxy @@ -170,6 +170,9 @@ and `proxy.upstreams[*].config` fields of the [proxy service definition](/docs/connect/registration/service-registration.html) that is actually registered. +To learn about other options that can be configured centrally see the +[Configuration Entries](/docs/agent/agent/config_entries.html) docs. + ### Proxy Config Options These fields may also be overridden explicitly in the [proxy service From aa60c1b63e36d62eb1bbc7ac7d6b801d4e678a77 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 23 Sep 2019 16:53:41 -0600 Subject: [PATCH 67/80] Fix broken link --- website/source/docs/connect/proxies/envoy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/connect/proxies/envoy.md b/website/source/docs/connect/proxies/envoy.md index 0a65472831b8..69ff2368dec2 100644 --- a/website/source/docs/connect/proxies/envoy.md +++ b/website/source/docs/connect/proxies/envoy.md @@ -171,7 +171,7 @@ definition](/docs/connect/registration/service-registration.html) that is actually registered. To learn about other options that can be configured centrally see the -[Configuration Entries](/docs/agent/agent/config_entries.html) docs. +[Configuration Entries](/docs/agent/config_entries.html) docs. ### Proxy Config Options From c2add7445375cc219716aab520bc2a57648bf883 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 23 Sep 2019 17:10:05 -0600 Subject: [PATCH 68/80] Fix TestManager_BasicLifecycle --- agent/proxycfg/manager_test.go | 6 ++++-- agent/proxycfg/testing.go | 27 +++++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/agent/proxycfg/manager_test.go b/agent/proxycfg/manager_test.go index f244acdec0cd..85c5e4d4559c 100644 --- a/agent/proxycfg/manager_test.go +++ b/agent/proxycfg/manager_test.go @@ -206,7 +206,8 @@ func TestManager_BasicLifecycle(t *testing.T) { WatchedGatewayEndpoints: map[string]map[string]structs.CheckServiceNodes{ "db": {}, }, - UpstreamEndpoints: map[string]structs.CheckServiceNodes{}, + UpstreamEndpoints: map[string]structs.CheckServiceNodes{}, + WatchedServiceChecks: map[string][]structs.CheckType{}, }, Datacenter: "dc1", }, @@ -250,7 +251,8 @@ func TestManager_BasicLifecycle(t *testing.T) { WatchedGatewayEndpoints: map[string]map[string]structs.CheckServiceNodes{ "db": {}, }, - UpstreamEndpoints: map[string]structs.CheckServiceNodes{}, + UpstreamEndpoints: map[string]structs.CheckServiceNodes{}, + WatchedServiceChecks: map[string][]structs.CheckType{}, }, Datacenter: "dc1", }, diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 7357f068dae2..b7eb1f806c96 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -20,12 +20,13 @@ import ( // TestCacheTypes encapsulates all the different cache types proxycfg.State will // watch/request for controlling one during testing. type TestCacheTypes struct { - roots *ControllableCacheType - leaf *ControllableCacheType - intentions *ControllableCacheType - health *ControllableCacheType - query *ControllableCacheType - compiledChain *ControllableCacheType + roots *ControllableCacheType + leaf *ControllableCacheType + intentions *ControllableCacheType + health *ControllableCacheType + query *ControllableCacheType + compiledChain *ControllableCacheType + serviceHTTPChecks *ControllableCacheType } // NewTestCacheTypes creates a set of ControllableCacheTypes for all types that @@ -33,12 +34,13 @@ type TestCacheTypes struct { func NewTestCacheTypes(t testing.T) *TestCacheTypes { t.Helper() ct := &TestCacheTypes{ - roots: NewControllableCacheType(t), - leaf: NewControllableCacheType(t), - intentions: NewControllableCacheType(t), - health: NewControllableCacheType(t), - query: NewControllableCacheType(t), - compiledChain: NewControllableCacheType(t), + roots: NewControllableCacheType(t), + leaf: NewControllableCacheType(t), + intentions: NewControllableCacheType(t), + health: NewControllableCacheType(t), + query: NewControllableCacheType(t), + compiledChain: NewControllableCacheType(t), + serviceHTTPChecks: NewControllableCacheType(t), } ct.query.blocking = false return ct @@ -76,6 +78,7 @@ func TestCacheWithTypes(t testing.T, types *TestCacheTypes) *cache.Cache { RefreshTimer: 0, RefreshTimeout: 10 * time.Minute, }) + c.RegisterType(cachetype.ServiceHTTPChecksName, types.serviceHTTPChecks, &cache.RegisterOptions{}) return c } From 9393b56ccced37cc65378cf836fe5ffbb14c71e1 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 24 Sep 2019 11:06:50 -0600 Subject: [PATCH 69/80] Add expose flag to service-reg full example --- .../docs/connect/registration/service-registration.html.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 2207173b2987..4fe64b313bfd 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -82,7 +82,8 @@ registering a proxy instance. "local_service_port": 9090, "config": {}, "upstreams": [], - "mesh_gateway": {} + "mesh_gateway": {}, + "expose": {} }, "port": 8181 } @@ -122,6 +123,10 @@ registering a proxy instance. - `mesh_gateway` `(object: {})` - Specifies the mesh gateway configuration for this proxy. The format is defined in the [Mesh Gateway Configuration Reference](#mesh-gateway-configuration-reference). + + - `expose` `(object: {})` - Specifies the configuration to expose HTTP paths through this proxy. + The format is defined in the [Expose Paths Configuration Reference](#expose-paths-configuration-reference), + and is only compatible with an Envoy proxy. ### Upstream Configuration Reference From e1a08c40ec52209c0c39a4ec96117edf99a27e4f Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 24 Sep 2019 11:43:05 -0600 Subject: [PATCH 70/80] Docs updates --- .../source/docs/agent/config-entries/proxy-defaults.html.md | 4 ++-- .../source/docs/agent/config-entries/service-defaults.html.md | 4 ++-- .../docs/connect/registration/service-registration.html.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/website/source/docs/agent/config-entries/proxy-defaults.html.md b/website/source/docs/agent/config-entries/proxy-defaults.html.md index 734398abc50b..205678e0101f 100644 --- a/website/source/docs/agent/config-entries/proxy-defaults.html.md +++ b/website/source/docs/agent/config-entries/proxy-defaults.html.md @@ -63,14 +63,14 @@ config { non-Connect-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. - - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running Consul on Kubernetes, and Consul agents run in their own pods. - `Paths` `array: []` - A list of paths to expose through Envoy. - - `Path` `(string: "")` - The HTTP path to expose. + - `Path` `(string: "")` - The HTTP path to expose. The path must be prefixed by a slash. ie: `/metrics`. - `LocalPathPort` `(int: 0)` - The port where the local service is listening for connections to the path. - `ListenerPort` `(int: 0)` - The port where the proxy will listen for connections. This port must be available for the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, diff --git a/website/source/docs/agent/config-entries/service-defaults.html.md b/website/source/docs/agent/config-entries/service-defaults.html.md index b3f73ce0b2d2..c232803d64cc 100644 --- a/website/source/docs/agent/config-entries/service-defaults.html.md +++ b/website/source/docs/agent/config-entries/service-defaults.html.md @@ -52,14 +52,14 @@ Protocol = "http" non-Connect-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. - - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running Consul on Kubernetes, and Consul agents run in their own pods. - `Paths` `array: []` - A list of paths to expose through Envoy. - - `Path` `(string: "")` - The HTTP path to expose. + - `Path` `(string: "")` - The HTTP path to expose. The path must be prefixed by a slash. ie: `/metrics`. - `LocalPathPort` `(int: 0)` - The port where the local service is listening for connections to the path. - `ListenerPort` `(int: 0)` - The port where the proxy will listen for connections. This port must be available for the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 4fe64b313bfd..1746477d20e1 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -300,14 +300,14 @@ registrations](/docs/agent/services.html#service-definition-parameter-case). } ``` -* `checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. +* `checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running Consul on Kubernetes, and Consul agents run in their own pods. * `paths` `array: []` - A list of paths to expose through Envoy. - - `path` `(string: "")` - The HTTP path to expose. + - `path` `(string: "")` - The HTTP path to expose. The path must be prefixed by a slash. ie: `/metrics`. - `local_path_port` `(int: 0)` - The port where the local service is listening for connections to the path. - `listener_port` `(int: 0)` - The port where the proxy will listen for connections. This port must be available for the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, From a97f4e61ab8e01937b7aa566bf142b9714c81532 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 24 Sep 2019 12:13:49 -0600 Subject: [PATCH 71/80] Update serviceConfigWatch ref --- agent/service_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/service_manager.go b/agent/service_manager.go index 29ea93ddc7cf..b3b22a3e184a 100644 --- a/agent/service_manager.go +++ b/agent/service_manager.go @@ -502,7 +502,7 @@ func (w *serviceConfigWatch) mergeServiceConfig() (*structs.NodeService, error) return nil, err } - if err := mergo.Merge(&ns.Proxy.Expose, s.defaults.Expose); err != nil { + if err := mergo.Merge(&ns.Proxy.Expose, w.defaults.Expose); err != nil { return nil, err } From 25712d05a703b36d4e695daa944f6d5f08c02ff9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 24 Sep 2019 12:14:35 -0600 Subject: [PATCH 72/80] Yank TLS flags --- agent/agent.go | 4 ++-- agent/agent_endpoint.go | 2 -- agent/agent_test.go | 6 +++--- agent/config/builder.go | 2 -- agent/config/config.go | 12 ------------ agent/structs/config_entry.go | 4 ---- agent/structs/connect_proxy_config.go | 16 --------------- agent/structs/structs.go | 11 ----------- agent/structs/structs_filtering_test.go | 20 ------------------- agent/structs/testing_catalog.go | 2 -- api/agent_test.go | 1 - api/config_entry.go | 12 ------------ api/config_entry_test.go | 20 ++----------------- command/config/write/config_write_test.go | 24 ++--------------------- 14 files changed, 9 insertions(+), 127 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 15078b4c8f4e..f2e5c318e85f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2331,8 +2331,8 @@ func (a *Agent) addServiceInternal(req *addServiceRequest) error { // Reset check targets if proxy was re-registered but no longer wants to expose checks // If the proxy is being registered for the first time then this is a no-op a.resetExposedChecks(service.Proxy.DestinationServiceID) - } - + } + if persistServiceConfig && a.config.DataDir != "" { var err error if persistDefaults != nil { diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 1816a2fa36cd..82fff12cd6e0 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -775,8 +775,6 @@ func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Re // Expose Config "local_path_port": "LocalPathPort", "listener_port": "ListenerPort", - "tls_skip_verify": "TLSSkipVerify", - "ca_file": "CAFile", // DON'T Recurse into these opaque config maps or we might mangle user's // keys. Note empty canonical is a special sentinel to prevent recursion. diff --git a/agent/agent_test.go b/agent/agent_test.go index 754269f576e3..5ebc4c304112 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3732,9 +3732,9 @@ func TestAgent_httpInjectAddr(t *testing.T) { got := httpInjectAddr(tt.url, tt.ip, tt.port) if got != tt.want { t.Errorf("httpInjectAddr() got = %v, want %v", got, tt.want) - } - }) - } + } + }) + } } func TestDefaultIfEmpty(t *testing.T) { diff --git a/agent/config/builder.go b/agent/config/builder.go index cd2c3261dcbb..472f8fa7a35c 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1373,8 +1373,6 @@ func (b *Builder) pathsVal(v []ExposePath) []structs.ExposePath { Path: b.stringVal(p.Path), LocalPathPort: b.intVal(p.LocalPathPort), Protocol: b.stringVal(p.Protocol), - TLSSkipVerify: b.boolVal(p.TLSSkipVerify), - CAFile: b.stringVal(p.CAFile), } } return paths diff --git a/agent/config/config.go b/agent/config/config.go index 449bba77b445..06d9bf0a9794 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -548,18 +548,6 @@ type ExposePath struct { // LocalPathPort is the port that the service is listening on for the given path. LocalPathPort *int `json:"local_path_port,omitempty" hcl:"local_path_port" mapstructure:"local_path_port"` - - // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. - TLSSkipVerify *bool `json:"tls_skip_verify,omitempty" hcl:"tls_skip_verify" mapstructure:"tls_skip_verify"` - - // CAFile is the path to the PEM encoded CA cert used to verify client certificates. - CAFile *string `json:"ca_file,omitempty" hcl:"ca_file" mapstructure:"ca_file"` - - // CertFile is the path to the PEM encoded CA cert used to verify client certificates. - CertFile *string `json:"cert_file,omitempty" hcl:"cert_file" mapstructure:"cert_file"` - - // KeyFile is the path to the PEM encoded CA cert used to verify client certificates. - KeyFile *string `json:"key_file,omitempty" hcl:"key_file" mapstructure:"key_file"` } // AutoEncrypt is the agent-global auto_encrypt configuration. diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 8bb834c4fa8e..b48c3407cb97 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -305,10 +305,6 @@ func ConfigEntryDecodeRulesForKind(kind string) (skipWhenPatching []string, tran }, map[string]string{ "local_path_port": "localpathport", "listener_port": "listenerport", - "key_file": "keyfile", - "cert_file": "certfile", - "ca_file": "cafile", - "tls_skip_verify": "tlsskipverify", "mesh_gateway": "meshgateway", "config": "", }, nil diff --git a/agent/structs/connect_proxy_config.go b/agent/structs/connect_proxy_config.go index 9e32ba7808d7..3921fc0f257a 100644 --- a/agent/structs/connect_proxy_config.go +++ b/agent/structs/connect_proxy_config.go @@ -348,18 +348,6 @@ type ExposePath struct { // Valid values are "http" and "http2", defaults to "http" Protocol string `json:",omitempty"` - // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. - TLSSkipVerify bool `json:",omitempty"` - - // CAFile is the path to the PEM encoded CA cert used to verify client certificates. - CAFile string `json:",omitempty"` - - // CertFile is the path fo the PEM encoded client certificate to authenticate Envoy - CertFile string `json:",omitempty"` - - // KeyFile is the path fo the PEM encoded client certificate to authenticate Envoy - KeyFile string `json:",omitempty"` - // ParsedFromCheck is set if this path was parsed from a registered check ParsedFromCheck bool } @@ -385,10 +373,6 @@ func (p *ExposePath) ToAPI() api.ExposePath { Path: p.Path, LocalPathPort: p.LocalPathPort, Protocol: p.Protocol, - TLSSkipVerify: p.TLSSkipVerify, - CAFile: p.CAFile, - CertFile: p.CertFile, - KeyFile: p.KeyFile, ParsedFromCheck: p.ParsedFromCheck, } } diff --git a/agent/structs/structs.go b/agent/structs/structs.go index f9e69c196389..9f03b55a5561 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -6,7 +6,6 @@ import ( "fmt" "math/rand" "net" - "os" "reflect" "regexp" "sort" @@ -996,16 +995,6 @@ func (s *NodeService) Validate() error { result = multierror.Append(result, fmt.Errorf("expose.paths: invalid listener port: %d", path.ListenerPort)) } - if path.CAFile != "" { - _, err := os.Stat(path.CAFile) - if err != nil { - result = multierror.Append(result, fmt.Errorf("expose.paths: failed to find CAFile '%s': %v", path.CAFile, err)) - } - } - if path.TLSSkipVerify == false && path.CAFile == "" { - result = multierror.Append(result, fmt.Errorf("expose.paths: CAFile must be provided if tls_skip_verify is false")) - } - path.Protocol = strings.ToLower(path.Protocol) if ok := allowedExposeProtocols[path.Protocol]; !ok && path.Protocol != "" { protocols := make([]string, 0) diff --git a/agent/structs/structs_filtering_test.go b/agent/structs/structs_filtering_test.go index 2cbf73aa57da..ca20fad2b491 100644 --- a/agent/structs/structs_filtering_test.go +++ b/agent/structs/structs_filtering_test.go @@ -94,26 +94,6 @@ var expectedFieldConfigPaths bexpr.FieldConfigurations = bexpr.FieldConfiguratio CoerceFn: bexpr.CoerceString, SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, }, - "TLSSkipVerify": &bexpr.FieldConfiguration{ - StructFieldName: "TLSSkipVerify", - CoerceFn: bexpr.CoerceBool, - SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, - }, - "CAFile": &bexpr.FieldConfiguration{ - StructFieldName: "CAFile", - CoerceFn: bexpr.CoerceString, - SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, - }, - "CertFile": &bexpr.FieldConfiguration{ - StructFieldName: "CertFile", - CoerceFn: bexpr.CoerceString, - SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, - }, - "KeyFile": &bexpr.FieldConfiguration{ - StructFieldName: "KeyFile", - CoerceFn: bexpr.CoerceString, - SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual, bexpr.MatchIn, bexpr.MatchNotIn, bexpr.MatchMatches, bexpr.MatchNotMatches}, - }, "ParsedFromCheck": &bexpr.FieldConfiguration{ StructFieldName: "ParsedFromCheck", CoerceFn: bexpr.CoerceBool, diff --git a/agent/structs/testing_catalog.go b/agent/structs/testing_catalog.go index 8c1a59bd5284..f25c4ef5bd07 100644 --- a/agent/structs/testing_catalog.go +++ b/agent/structs/testing_catalog.go @@ -64,13 +64,11 @@ func TestNodeServiceExpose(t testing.T) *NodeService { Path: "/foo", LocalPathPort: 8080, ListenerPort: 21500, - TLSSkipVerify: true, }, { Path: "/bar", LocalPathPort: 8080, ListenerPort: 21501, - TLSSkipVerify: true, }, }, }, diff --git a/api/agent_test.go b/api/agent_test.go index 4a066e667666..3b8e61f4a7b0 100644 --- a/api/agent_test.go +++ b/api/agent_test.go @@ -1580,7 +1580,6 @@ func TestAgentService_ExposeChecks(t *testing.T) { ListenerPort: 21500, Path: "/metrics", Protocol: "http2", - TLSSkipVerify: true, } reg := AgentServiceRegistration{ Kind: ServiceKindConnectProxy, diff --git a/api/config_entry.go b/api/config_entry.go index 31d51fd872c6..5c05311be490 100644 --- a/api/config_entry.go +++ b/api/config_entry.go @@ -81,18 +81,6 @@ type ExposePath struct { // Valid values are "http" and "http2", defaults to "http" Protocol string `json:",omitempty"` - // TLSSkipVerify defines whether incoming requests should be authenticated with TLS. - TLSSkipVerify bool `json:",omitempty"` - - // CAFile is the path to the PEM encoded CA cert used to verify client certificates. - CAFile string `json:",omitempty"` - - // CertFile is the path fo the PEM encoded client certificate to authenticate Envoy - CertFile string `json:",omitempty"` - - // KeyFile is the path fo the PEM encoded client certificate to authenticate Envoy - KeyFile string `json:",omitempty"` - // ParsedFromCheck is set if this path was parsed from a registered check ParsedFromCheck bool } diff --git a/api/config_entry_test.go b/api/config_entry_test.go index 3d32ea7300af..893d024a2bf6 100644 --- a/api/config_entry_test.go +++ b/api/config_entry_test.go @@ -208,11 +208,7 @@ func TestDecodeConfigEntry(t *testing.T) { "LocalPathPort": 8080, "ListenerPort": 21500, "Path": "/healthz", - "Protocol": "http2", - "TLSSkipVerify": true, - "CAFile": "ca.pem", - "CertFile": "cert.pem", - "KeyFile": "key.pem" + "Protocol": "http2" } ] } @@ -229,10 +225,6 @@ func TestDecodeConfigEntry(t *testing.T) { ListenerPort: 21500, Path: "/healthz", Protocol: "http2", - TLSSkipVerify: true, - CAFile: "ca.pem", - CertFile: "cert.pem", - KeyFile: "key.pem", }, }, }, @@ -251,11 +243,7 @@ func TestDecodeConfigEntry(t *testing.T) { "LocalPathPort": 8080, "ListenerPort": 21500, "Path": "/healthz", - "Protocol": "http2", - "TLSSkipVerify": true, - "CAFile": "ca.pem", - "CertFile": "cert.pem", - "KeyFile": "key.pem" + "Protocol": "http2" } ] } @@ -272,10 +260,6 @@ func TestDecodeConfigEntry(t *testing.T) { ListenerPort: 21500, Path: "/healthz", Protocol: "http2", - TLSSkipVerify: true, - CAFile: "ca.pem", - CertFile: "cert.pem", - KeyFile: "key.pem", }, }, }, diff --git a/command/config/write/config_write_test.go b/command/config/write/config_write_test.go index e31dcc14df18..1e1bc2e61784 100644 --- a/command/config/write/config_write_test.go +++ b/command/config/write/config_write_test.go @@ -1055,10 +1055,6 @@ func TestParseConfigEntry(t *testing.T) { listener_port = 21500 path = "/healthz" protocol = "http2" - tls_skip_verify = true - ca_file = "ca.pem" - cert_file = "cert.pem" - key_file = "key.pem" } ] }`, @@ -1073,10 +1069,6 @@ func TestParseConfigEntry(t *testing.T) { ListenerPort = 21500 Path = "/healthz" Protocol = "http2" - TLSSkipVerify = true - CAFile = "ca.pem" - CertFile = "cert.pem" - KeyFile = "key.pem" } ] }`, @@ -1091,11 +1083,7 @@ func TestParseConfigEntry(t *testing.T) { "local_path_port": 8080, "listener_port": 21500, "path": "/healthz", - "protocol": "http2", - "tls_skip_verify": true, - "ca_file": "ca.pem", - "cert_file": "cert.pem", - "key_file": "key.pem" + "protocol": "http2" } ] } @@ -1112,11 +1100,7 @@ func TestParseConfigEntry(t *testing.T) { "LocalPathPort": 8080, "ListenerPort": 21500, "Path": "/healthz", - "Protocol": "http2", - "TLSSkipVerify": true, - "CAFile": "ca.pem", - "CertFile": "cert.pem", - "KeyFile": "key.pem" + "Protocol": "http2" } ] } @@ -1133,10 +1117,6 @@ func TestParseConfigEntry(t *testing.T) { Path: "/healthz", LocalPathPort: 8080, Protocol: "http2", - TLSSkipVerify: true, - CAFile: "ca.pem", - CertFile: "cert.pem", - KeyFile: "key.pem", }, }, }, From 091d771446c4ffd3d7f26a7fa338626fad3db5e9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 24 Sep 2019 13:16:03 -0600 Subject: [PATCH 73/80] go vet fix --- agent/config/runtime_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 82a0b0e8c5fc..a6109d8c2a9d 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -3976,8 +3976,7 @@ func TestFullConfig(t *testing.T) { "path": "/health", "local_path_port": 8080, "listener_port": 21500, - "protocol": "http", - "tls_skip_verify": true + "protocol": "http" } ] }, @@ -4592,7 +4591,6 @@ func TestFullConfig(t *testing.T) { local_path_port = 8080 listener_port = 21500 protocol = "http" - tls_skip_verify = true } ] } @@ -5178,7 +5176,6 @@ func TestFullConfig(t *testing.T) { LocalPathPort: 8080, ListenerPort: 21500, Protocol: "http", - TLSSkipVerify: true, }, }, }, From b10c8bf4341f70c18ee6dd6ce13fdddb950acd15 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 24 Sep 2019 13:20:08 -0600 Subject: [PATCH 74/80] Fix docs example --- website/source/docs/agent/services.html.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/website/source/docs/agent/services.html.md b/website/source/docs/agent/services.html.md index fc24abd9b4ef..9d588b7761ac 100644 --- a/website/source/docs/agent/services.html.md +++ b/website/source/docs/agent/services.html.md @@ -72,11 +72,9 @@ example shows all possible fields, but note that only a few are required. "paths": [ { "path": "/healthz", - "protocol": "http2", - "tls_skip_verify": false, - "key_file": "key.pem", - "cert_file": "cert.pem", - "ca_file": "ca.pem" + "local_path_port": 8080, + "listener_port": 21500, + "protocol": "http2" } ] } From 717b68600a80801667cd358ab6bd802045995034 Mon Sep 17 00:00:00 2001 From: Freddy Date: Wed, 25 Sep 2019 10:52:15 -0600 Subject: [PATCH 75/80] Update website/source/docs/connect/registration/service-registration.html.md Co-Authored-By: Judith Malnick --- .../docs/connect/registration/service-registration.html.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 1746477d20e1..6db8434a656b 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -254,7 +254,7 @@ Exposing paths through Envoy enables a service to protect itself by only listeni non-Connect-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. -Note that `snake_case` is used here as it works in both [config file and API +-> Note that `snake_case` is used here as it works in both [config file and API registrations](/docs/agent/services.html#service-definition-parameter-case). #### Expose listeners in Envoy for HTTP and GRPC checks registered with the local Consul agent @@ -312,4 +312,4 @@ registrations](/docs/agent/services.html#service-definition-parameter-case). - `listener_port` `(int: 0)` - The port where the proxy will listen for connections. This port must be available for the listener to be set up. If the port is not free then Envoy will not expose a listener for the path, but the proxy registration will not fail. - - `protocol` `(string: "http")` - Sets the protocol of the listener. One of `http` or `http2`. For gRPC use `http2`. \ No newline at end of file + - `protocol` `(string: "http")` - Sets the protocol of the listener. One of `http` or `http2`. For gRPC use `http2`. From e30cb8107bddfbbeb69dacb366ed0702b1529bc2 Mon Sep 17 00:00:00 2001 From: Freddy Date: Wed, 25 Sep 2019 10:52:24 -0600 Subject: [PATCH 76/80] Update website/source/docs/connect/registration/service-registration.html.md Co-Authored-By: Judith Malnick --- .../docs/connect/registration/service-registration.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 6db8434a656b..3c272daea6fe 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -302,7 +302,7 @@ registrations](/docs/agent/services.html#service-definition-parameter-case). * `checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's - [advertise address](/docs/agent/options.html#advertise).The port for these listeners are dynamically allocated from + [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). This flag is useful when a Consul client cannot reach registered services over localhost. One example is when running Consul on Kubernetes, and Consul agents run in their own pods. From ff5ca75982fa8ed9a976285883a2b5bb90b423f3 Mon Sep 17 00:00:00 2001 From: Freddy Date: Wed, 25 Sep 2019 10:52:48 -0600 Subject: [PATCH 77/80] Update website/source/docs/connect/registration/service-registration.html.md Co-Authored-By: Judith Malnick --- .../docs/connect/registration/service-registration.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 3c272daea6fe..2c0214c19375 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -300,7 +300,7 @@ registrations](/docs/agent/services.html#service-definition-parameter-case). } ``` -* `checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. +* `checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). From 8d68b38ef720c4483fa706e7f8613d56e405331d Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 25 Sep 2019 11:00:57 -0600 Subject: [PATCH 78/80] Final docs updates --- .../source/docs/agent/config-entries/proxy-defaults.html.md | 2 +- .../source/docs/agent/config-entries/service-defaults.html.md | 2 +- .../docs/connect/registration/service-registration.html.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/website/source/docs/agent/config-entries/proxy-defaults.html.md b/website/source/docs/agent/config-entries/proxy-defaults.html.md index 205678e0101f..33a154f39df6 100644 --- a/website/source/docs/agent/config-entries/proxy-defaults.html.md +++ b/website/source/docs/agent/config-entries/proxy-defaults.html.md @@ -63,7 +63,7 @@ config { non-Connect-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. - - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). diff --git a/website/source/docs/agent/config-entries/service-defaults.html.md b/website/source/docs/agent/config-entries/service-defaults.html.md index c232803d64cc..470b28b4b582 100644 --- a/website/source/docs/agent/config-entries/service-defaults.html.md +++ b/website/source/docs/agent/config-entries/service-defaults.html.md @@ -52,7 +52,7 @@ Protocol = "http" non-Connect-enabled applications to contact an HTTP endpoint. Some examples include: exposing a `/metrics` path for Prometheus or `/healthz` for kubelet liveness checks. - - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. + - `Checks` `(bool: false)` - If enabled, all HTTP and gRPC checks registered with the agent are exposed through Envoy. Envoy will expose listeners for these checks and will only accept connections originating from localhost or Consul's [advertise address](/docs/agent/options.html#advertise). The port for these listeners are dynamically allocated from [expose_min_port](/docs/agent/options.html#expose_min_port) to [expose_max_port](/docs/agent/options.html#expose_max_port). diff --git a/website/source/docs/connect/registration/service-registration.html.md b/website/source/docs/connect/registration/service-registration.html.md index 2c0214c19375..b000bf3601a6 100644 --- a/website/source/docs/connect/registration/service-registration.html.md +++ b/website/source/docs/connect/registration/service-registration.html.md @@ -132,7 +132,7 @@ registering a proxy instance. The following examples show all possible upstream configuration parameters. -Note that `snake_case` is used here as it works in both [config file and API +-> Note that `snake_case` is used here as it works in both [config file and API registrations](/docs/agent/services.html#service-definition-parameter-case). Upstreams support multiple destination types. Both examples are shown below @@ -199,7 +199,7 @@ followed by documentation for each attribute. The following examples show all possible mesh gateway configurations. -Note that `snake_case` is used here as it works in both [config file and API +-> Note that `snake_case` is used here as it works in both [config file and API registrations](/docs/agent/services.html#service-definition-parameter-case). #### Using a Local/Egress Gateway in the Local Datacenter From 1c00dc365f8635507648c48edc28af0712fb8d5f Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 25 Sep 2019 16:58:32 -0600 Subject: [PATCH 79/80] Fix http2 integration test --- agent/xds/listeners.go | 6 +----- test/integration/connect/envoy/case-http2/capture.sh | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100755 test/integration/connect/envoy/case-http2/capture.sh diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 601e8d9f065b..d7b1dee975b5 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -655,14 +655,10 @@ func makeHTTPFilter( if grpc { proto = "grpc" } - codec := envoyhttp.AUTO - if grpc || http2 { - codec = envoyhttp.HTTP2 - } cfg := &envoyhttp.HttpConnectionManager{ StatPrefix: makeStatPrefix(proto, statPrefix, filterName), - CodecType: codec, + CodecType: envoyhttp.AUTO, HttpFilters: []*envoyhttp.HttpFilter{ &envoyhttp.HttpFilter{ Name: "envoy.router", diff --git a/test/integration/connect/envoy/case-http2/capture.sh b/test/integration/connect/envoy/case-http2/capture.sh new file mode 100755 index 000000000000..1a11f7d5e014 --- /dev/null +++ b/test/integration/connect/envoy/case-http2/capture.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:19000 s1 primary || true +snapshot_envoy_admin localhost:19001 s2 primary || true From 18638a40f6f79b8ffbc2c46a7432e411ff183d05 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 25 Sep 2019 17:06:22 -0600 Subject: [PATCH 80/80] Update golden files --- .../listeners/connect-proxy-with-chain-and-overrides.golden | 1 - .../xds/testdata/listeners/connect-proxy-with-grpc-chain.golden | 1 - .../xds/testdata/listeners/connect-proxy-with-http2-chain.golden | 1 - .../xds/testdata/listeners/expose-paths-new-cluster-http2.golden | 1 - 4 files changed, 4 deletions(-) diff --git a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden index 41269e441840..a911cd1890b3 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-chain-and-overrides.golden @@ -16,7 +16,6 @@ { "name": "envoy.http_connection_manager", "config": { - "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden b/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden index 41269e441840..a911cd1890b3 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-grpc-chain.golden @@ -16,7 +16,6 @@ { "name": "envoy.http_connection_manager", "config": { - "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden b/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden index 0445877995cb..6994537d4f4e 100644 --- a/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden +++ b/agent/xds/testdata/listeners/connect-proxy-with-http2-chain.golden @@ -16,7 +16,6 @@ { "name": "envoy.http_connection_manager", "config": { - "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [ diff --git a/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden b/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden index 633f3c529fab..1d9afe43568b 100644 --- a/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden +++ b/agent/xds/testdata/listeners/expose-paths-new-cluster-http2.golden @@ -16,7 +16,6 @@ { "name": "envoy.http_connection_manager", "config": { - "codec_type": "HTTP2", "http2_protocol_options": { }, "http_filters": [