Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

xDS Mesh Gateway Resolver Subset Fixes #7294

Merged
merged 2 commits into from
Feb 19, 2020
Merged

xDS Mesh Gateway Resolver Subset Fixes #7294

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d9bffe7
xDS Mesh Gateway Resolver Subset Fixes
mkeeler Feb 13, 2020
01f85b7
Refactor the gateway endpoint generation to be a little easier to read
mkeeler Feb 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions agent/proxycfg/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,25 @@ func (c *configSnapshotConnectProxy) IsEmpty() bool {
}

type configSnapshotMeshGateway struct {
WatchedServices map[structs.ServiceID]context.CancelFunc
// map of service id to a cancel function. This cancel function is tied to the watch of
// connect enabled services for the given id. If the main datacenter services watch would
// indicate the removal of a service all together we then cancel watching that service for
// its connect endpoints.
WatchedServices map[structs.ServiceID]context.CancelFunc
// Indicates that the watch on the datacenters services has completed. Even when there
// are no connect services, this being set (and the Connect roots being available) will be enough for
// the config snapshot to be considered valid. In the case of Envoy, this allows it to start its listeners
// even when no services would be proxied and allow its health check to pass.
WatchedServicesSet bool
// map of datacenter name to a cancel function. This cancel function is tied
// to the watch of mesh-gateway services in that datacenter.
WatchedDatacenters map[string]context.CancelFunc
ServiceGroups map[structs.ServiceID]structs.CheckServiceNodes
ServiceResolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry
GatewayGroups map[string]structs.CheckServiceNodes
// map of service id to the service instances of that service in the local datacenter
ServiceGroups map[structs.ServiceID]structs.CheckServiceNodes
// map of service id to an associated service-resolver config entry for that service
ServiceResolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry
// map of datacenter names to services of kind mesh-gateway in that datacenter
GatewayGroups map[string]structs.CheckServiceNodes
}

func (c *configSnapshotMeshGateway) IsEmpty() bool {
Expand Down
19 changes: 10 additions & 9 deletions agent/xds/clusters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,18 +136,19 @@ func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsho
return nil, err
}
clusters = append(clusters, cluster)
}

// generate the service subset clusters
for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
for subsetName, _ := range resolver.Subsets {
clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
// if there is a service-resolver for this service then also setup subset clusters for it
if resolver, ok := cfgSnap.MeshGateway.ServiceResolvers[svc]; ok {
// generate 1 cluster for each service subset
for subsetName, _ := range resolver.Subsets {
clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)

cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
if err != nil {
return nil, err
cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
if err != nil {
return nil, err
}
clusters = append(clusters, cluster)
}
clusters = append(clusters, cluster)
}
}

Expand Down
36 changes: 36 additions & 0 deletions agent/xds/clusters_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,42 @@ func TestClustersFromSnapshot(t *testing.T) {
}
},
},
{
name: "mesh-gateway-ignore-extra-resolvers",
create: proxycfg.TestConfigSnapshotMeshGateway,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.MeshGateway.ServiceResolvers = map[structs.ServiceID]*structs.ServiceResolverConfigEntry{
structs.NewServiceID("bar", nil): &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "bar",
DefaultSubset: "v2",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": structs.ServiceResolverSubset{
Filter: "Service.Meta.Version == 1",
},
"v2": structs.ServiceResolverSubset{
Filter: "Service.Meta.Version == 2",
OnlyPassing: true,
},
},
},
structs.NewServiceID("notfound", nil): &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "notfound",
DefaultSubset: "v2",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": structs.ServiceResolverSubset{
Filter: "Service.Meta.Version == 1",
},
"v2": structs.ServiceResolverSubset{
Filter: "Service.Meta.Version == 2",
OnlyPassing: true,
},
},
},
}
},
},
}

for _, tt := range tests {
Expand Down
87 changes: 51 additions & 36 deletions agent/xds/endpoints.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,52 +165,67 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
resources = append(resources, la)
}

// generate the endpoints for the local service groups
// Generate the endpoints for each service and its subsets
for svc, endpoints := range cfgSnap.MeshGateway.ServiceGroups {
defaultSubsetEndpoints := endpoints
defaultSubsetOnlyPassing := false
clusterName := connect.ServiceSNI(svc.ID, "", svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
la := makeLoadAssignment(
clusterName,
[]loadAssignmentEndpointGroup{
{Endpoints: endpoints},
},
cfgSnap.Datacenter,
)
resources = append(resources, la)
}

// generate the endpoints for the service subsets
for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
for subsetName, subset := range resolver.Subsets {
clusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)

endpoints := cfgSnap.MeshGateway.ServiceGroups[svc]
// Generate endpoints for each subset cluster. We do this before generating the endpoints for the default/unnamed
// subset so that we can take into account the DefaultSubset on the service resolver which may prevent the default/unnamed
// cluster from creating endpoints for all service instances.
if resolver, hasResolver := cfgSnap.MeshGateway.ServiceResolvers[svc]; hasResolver {
for subsetName, subset := range resolver.Subsets {
subsetEndpoints := endpoints

// locally execute the subsets filter
if subset.Filter != "" {
filter, err := bexpr.CreateFilter(subset.Filter, nil, endpoints)
if err != nil {
return nil, err
}

// locally execute the subsets filter
if subset.Filter != "" {
filter, err := bexpr.CreateFilter(subset.Filter, nil, endpoints)
if err != nil {
return nil, err
raw, err := filter.Execute(endpoints)
if err != nil {
return nil, err
}
subsetEndpoints = raw.(structs.CheckServiceNodes)
}

raw, err := filter.Execute(endpoints)
if err != nil {
return nil, err
subsetClusterName := connect.ServiceSNI(svc.ID, subsetName, svc.NamespaceOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
la := makeLoadAssignment(
subsetClusterName,
[]loadAssignmentEndpointGroup{
{
Endpoints: subsetEndpoints,
OnlyPassing: subset.OnlyPassing,
},
},
cfgSnap.Datacenter,
)
resources = append(resources, la)

// if this is the resolvers default subset then override
// how the default/unnamed clusters endpoints will be configured
if subsetName == resolver.DefaultSubset {
defaultSubsetEndpoints = subsetEndpoints
defaultSubsetOnlyPassing = subset.OnlyPassing
}
endpoints = raw.(structs.CheckServiceNodes)
}
}

la := makeLoadAssignment(
clusterName,
[]loadAssignmentEndpointGroup{
{
Endpoints: endpoints,
OnlyPassing: subset.OnlyPassing,
},
// create the load assignment for the default
la := makeLoadAssignment(
clusterName,
[]loadAssignmentEndpointGroup{
{
Endpoints: defaultSubsetEndpoints,
OnlyPassing: defaultSubsetOnlyPassing,
},
cfgSnap.Datacenter,
)
resources = append(resources, la)
}
},
cfgSnap.Datacenter,
)
resources = append(resources, la)
}

return resources, nil
Expand Down
36 changes: 36 additions & 0 deletions agent/xds/endpoints_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,42 @@ func Test_endpointsFromSnapshot(t *testing.T) {
}
},
},
{
name: "mesh-gateway-default-service-subset",
create: proxycfg.TestConfigSnapshotMeshGateway,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.MeshGateway.ServiceResolvers = map[structs.ServiceID]*structs.ServiceResolverConfigEntry{
structs.NewServiceID("bar", nil): &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "bar",
DefaultSubset: "v2",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": structs.ServiceResolverSubset{
Filter: "Service.Meta.version == 1",
},
"v2": structs.ServiceResolverSubset{
Filter: "Service.Meta.version == 2",
OnlyPassing: true,
},
},
},
structs.NewServiceID("foo", nil): &structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
DefaultSubset: "v2",
Subsets: map[string]structs.ServiceResolverSubset{
"v1": structs.ServiceResolverSubset{
Filter: "Service.Meta.version == 1",
},
"v2": structs.ServiceResolverSubset{
Filter: "Service.Meta.version == 2",
OnlyPassing: true,
},
},
},
}
},
},
}

for _, tt := range tests {
Expand Down
87 changes: 87 additions & 0 deletions agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

}
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

}
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

}
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

}
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
},
{
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

}
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Cluster",
"nonce": "00000001"
}
Loading