From f9d9c930533cf4a7843ac9896eced95b9c8d341f Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 5 Aug 2018 15:01:26 -0300 Subject: [PATCH 1/4] Add blue/green balance mode --- README.md | 31 ++-- examples/blue-green/README.md | 140 ++++++++++++++---- .../ingress/annotations/bluegreen/main.go | 52 +++++-- pkg/common/ingress/controller/controller.go | 93 ++++++++++-- pkg/common/ingress/types.go | 3 +- pkg/common/utils/utils.go | 14 ++ pkg/common/utils/utils_test.go | 46 ++++++ 7 files changed, 322 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 964774391..576b65c13 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ The following annotations are supported: ||[`ingress.kubernetes.io/auth-tls-secret`](#auth-tls)|namespace/secret name|[doc](/examples/auth/client-certs)| |`[0]`|[`ingress.kubernetes.io/balance-algorithm`](#balance-algorithm)|algorithm name|-| |`[0]`|[`ingress.kubernetes.io/blue-green-deploy`](#blue-green)|label=value=weight,...|[doc](/examples/blue-green)| +|`[1]`|[`ingress.kubernetes.io/blue-green-balance`](#blue-green)|label=value=weight,...|[doc](/examples/blue-green)| +|`[1]`|[`ingress.kubernetes.io/blue-green-mode`](#blue-green)|[pod\|deploy]|[doc](/examples/blue-green)| |`[0]`|[`ingress.kubernetes.io/config-backend`](#configuration-snippet)|multiline HAProxy backend config|-| |`[0]`|[`ingress.kubernetes.io/cors-allow-origin`](#cors)|URL|-| |`[0]`|[`ingress.kubernetes.io/cors-allow-methods`](#cors)|methods list|-| @@ -132,25 +134,34 @@ See also client cert [sample](/examples/auth/client-certs). Configure weight of a blue/green deployment. The annotation accepts a comma separated list of label name/value pair and a numeric weight. Concatenate label name, label value and weight with an equal -sign, without spaces. The label name/value pair will be used to match corresponding pods. +sign, without spaces. The label name/value pair will be used to match corresponding pods or deploys. +There is no limit to the number of label/weight balance configurations. The endpoints of a single backend are selected using service selectors, which also uses labels. Because of that, in order to use blue/green deployment, the deployment, daemon set or replication controller template should have at least two label name/value pairs - one that matches the service selector and another that matches the blue/green selector. +* `ingress.kubernetes.io/blue-green-balance`: comma separated list of labels and weights +* `ingress.kubernetes.io/blue-green-deploy`: deprecated on v0.7, this is an alias to `ingress.kubernetes.io/blue-green-balance`. +* `ingress.kubernetes.io/blue-green-mode`: how to apply the weights, might be `pod` or `deploy` + The following configuration `group=blue=1,group=green=4` will redirect 20% of the load to the -`group=blue` pods and 80% of the load to the `group=green` if they have the same number of replicas. +`group=blue` group and 80% of the load to `group=green` group. + +Applying the weights depends on the blue/green mode. v0.6 has only `pod` mode which means that +every single pod receives the same weight as configured on blue/green balance. This means that +a balance configuration with 50% to each group will redirect twice as much requests to a backend +that has the double of replicas. v0.7 has also `deploy` mode which rebalance the weights based +on the number of replicas of each deployment. -Note that this configuration is related to every single pod. On the configuration above, if -`group=blue` has two replicas and `group=green` has just one, green would receive only the double -of the number of requests dedicated to blue. This can be adjusted using higher numbers - eg `10/40` -instead of `1/4` - and divided by the number of replicas of each deployment - eg `5/40` instead of -`10/40`. +In short, regarding blue/green mode: use `pod` if you want to redirect more requests to a +deployment updating the number of replicas; use `deploy` if you want to control the load +of each side updating the blue/green balance annotation. -Value of `0` (zero) can also be used. This will let the endpoint configured in the backend accepting -persistent connections - see [affinity](#affinity) - but will not participate in the load balancing. -The maximum weight value is `256`. +Value of `0` (zero) can also be used as weight. This will let the endpoint configured in the +backend accepting persistent connections - see [affinity](#affinity) - but will not participate +in the load balancing. The maximum weight value is `256`. See also the [example](/examples/blue-green) page. diff --git a/examples/blue-green/README.md b/examples/blue-green/README.md index c61d836a3..e3897eb22 100644 --- a/examples/blue-green/README.md +++ b/examples/blue-green/README.md @@ -20,17 +20,17 @@ and `group` label as the blue/green deployment selector: ``` $ kubectl run blue \ - --image=gcr.io/google_containers/echoserver:1.3 \ - --port=8080 --labels=run=bluegreen,group=blue + --image=jcmoraisjr/whoami \ + --port=8000 --labels=run=bluegreen,group=blue deployment "blue" created $ kubectl run green \ - --image=gcr.io/google_containers/echoserver:1.3 \ - --port=8080 --labels=run=bluegreen,group=green + --image=jcmoraisjr/whoami \ + --port=8000 --labels=run=bluegreen,group=green deployment "green" created ``` -Certify if the pods are running and have the correct labels. Note that both `group` and `run` +Certify that the pods are running and have the correct labels. Note that both `group` and `run` labels were applied: ``` @@ -40,7 +40,7 @@ blue-79c9b67d5b-5hd2r 1/1 Running 0 35s group=blue,pod green-7546d648c4-p7pmz 1/1 Running 0 28s group=green,pod-template-hash=3102820470,run=bluegreen ``` -# Configure +## Configure Create a service that bind both deployments together using the `run` label. The expose command need a deployment object, take anyone, we will override it's selector: @@ -58,7 +58,7 @@ Check also the endpoints, it should list both blue and green pods: ``` $ kubectl get ep bluegreen NAME ENDPOINTS AGE -bluegreen 172.17.0.11:8080,172.17.0.19:8080 2m +bluegreen 172.17.0.11:8000,172.17.0.19:8000 2m $ kubectl get pod -lrun=bluegreen -owide NAME READY STATUS RESTARTS AGE IP NODE @@ -66,7 +66,7 @@ blue-79c9b67d5b-5hd2r 1/1 Running 0 2m 172.17.0.11 green-7546d648c4-p7pmz 1/1 Running 0 2m 172.17.0.19 192.168.100.99 ``` -Configure the ingress resource: +Configure the ingress resource. No need to change the host below, `bluegreen.example.com` is fine: ``` $ kubectl create -f - </dev/null - done -.................................................. +$ kubectl annotate --overwrite ingress bluegreen \ + ingress.kubernetes.io/blue-green-mode=deploy ``` -Now check `:1936` - the stats page has a `-bluegreen-8080` card -with backend stats. The column Session/Total should have about 20% of the deployments -on one endpoint and 80% on another. +* BG Mode: deploy +* BG Balance: blue=1, green=1 +* Replicas: blue=1, green=3 -The blue/green configuration is related to every single pod - if the number of replicas change, -the load will also change between blue and green groups. Have a look at the -[doc](/README.md#blue-green) on how it works. +``` +$ hareq +Running 100 requests... + 50 blue + 50 green +``` + +--- + +Changing now the balance to 1/3 blue and 2/3 green: + +``` +$ kubectl annotate --overwrite ingress bluegreen \ + ingress.kubernetes.io/blue-green-balance=group=blue=1,group=green=2 +``` + +* BG Mode: deploy +* BG Balance: blue=1, green=2 +* Replicas: blue=1, green=3 + +``` +$ hareq +Running 100 requests... + 33 blue + 67 green +``` + +--- + +The balance will be the same despite the number of replicas: + +``` +$ kubectl scale deploy green --replicas=6 +$ kubectl get pod -w +``` + +* BG Mode: deploy +* BG Balance: blue=1, green=2 +* Replicas: blue=1, green=6 + +``` +$ hareq +Running 100 requests... + 33 blue + 67 green +``` diff --git a/pkg/common/ingress/annotations/bluegreen/main.go b/pkg/common/ingress/annotations/bluegreen/main.go index 12b2422a0..65c4ad93a 100644 --- a/pkg/common/ingress/annotations/bluegreen/main.go +++ b/pkg/common/ingress/annotations/bluegreen/main.go @@ -18,26 +18,37 @@ package bluegreen import ( "fmt" + "github.com/golang/glog" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/parser" extensions "k8s.io/api/extensions/v1beta1" + "regexp" "strconv" "strings" ) const ( - blueGreenAnn = "ingress.kubernetes.io/blue-green-deploy" + blueGreenBalanceAnn = "ingress.kubernetes.io/blue-green-balance" + blueGreenDeployAnn = "ingress.kubernetes.io/blue-green-deploy" + blueGreenModeAnn = "ingress.kubernetes.io/blue-green-mode" +) + +var ( + modeAnnRegex = regexp.MustCompile(`^(pod|deploy)$`) ) // DeployWeight has one label name/value pair and it's weight type DeployWeight struct { - LabelName string - LabelValue string - Weight int + LabelName string + LabelValue string + PodWeight int + PodCount int + GroupWeight int } // Config is the blue/green deployment configuration type Config struct { DeployWeight []DeployWeight + Mode string } type bgdeploy struct { @@ -50,9 +61,12 @@ func NewParser() parser.IngressAnnotation { // Parse parses blue/green annotation and create a Config struct func (bg bgdeploy) Parse(ing *extensions.Ingress) (interface{}, error) { - s, err := parser.GetStringAnnotation(blueGreenAnn, ing) + s, err := parser.GetStringAnnotation(blueGreenBalanceAnn, ing) if err != nil { - return nil, err + s, _ = parser.GetStringAnnotation(blueGreenDeployAnn, ing) + if s == "" { + return nil, err + } } weights := strings.Split(s, ",") var dw []DeployWeight @@ -66,22 +80,40 @@ func (bg bgdeploy) Parse(ing *extensions.Ingress) (interface{}, error) { return nil, fmt.Errorf("error reading blue/green config: %v", err) } if w < 0 { + glog.Warningf("invalid weight '%v' on '%v/%v', using '0'", w, ing.Namespace, ing.Name) w = 0 } + if w > 256 { + glog.Warningf("invalid weight '%v' on '%v/%v', using '256'", w, ing.Namespace, ing.Name) + w = 256 + } dwItem := DeployWeight{ - LabelName: dwSlice[0], - LabelValue: dwSlice[1], - Weight: int(w), + LabelName: dwSlice[0], + LabelValue: dwSlice[1], + PodWeight: int(w), + PodCount: 0, // updated in the controller + GroupWeight: 0, // updated in the controller } dw = append(dw, dwItem) } + mode, _ := parser.GetStringAnnotation(blueGreenModeAnn, ing) + if !modeAnnRegex.MatchString(mode) { + if mode != "" { + glog.Warningf("unsupported blue/green mode '%v' on '%v/%v', falling back to 'pod'", mode, ing.Namespace, ing.Name) + } + mode = "pod" + } return &Config{ DeployWeight: dw, + Mode: mode, }, nil } // Equal tests equality between two Config objects func (b1 *Config) Equal(b2 *Config) bool { + if b1.Mode != b2.Mode { + return false + } if len(b1.DeployWeight) != len(b2.DeployWeight) { return false } @@ -108,7 +140,7 @@ func (dw1 *DeployWeight) Equal(dw2 *DeployWeight) bool { if dw1.LabelValue != dw2.LabelValue { return false } - if dw1.Weight != dw2.Weight { + if dw1.PodWeight != dw2.PodWeight { return false } return true diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index 096280a79..cbb7fc56b 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -43,6 +43,7 @@ import ( "k8s.io/client-go/util/flowcontrol" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/class" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/healthcheck" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/parser" @@ -689,6 +690,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) svc := upstream.Service podNamespace := svc.Namespace deployWeight := upstream.BlueGreen.DeployWeight + hasBlueGreenDeploy := false for epID := range upstream.Endpoints { ep := &upstream.Endpoints[epID] if ep.Draining { @@ -703,18 +705,22 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) } if ep.Target == nil { glog.Warningf("ignoring blue/green config due to empty object reference on endpoint %v/%v", podNamespace, upstream.Name) + ep.Weight = 1 continue } podName := ep.Target.Name - weight := -1 + var weightRef *bluegreen.DeployWeight if pod, err := ic.listers.Pod.GetPod(podNamespace, podName); err == nil { - for _, weightConfig := range deployWeight { + for wID := range deployWeight { + weightConfig := &deployWeight[wID] if label, found := pod.Labels[weightConfig.LabelName]; found { if label == weightConfig.LabelValue { - if weight < 0 { - weight = weightConfig.Weight - } else if weightConfig.Weight != weight { - glog.Warningf("deployment weight %v to service %v/%v is duplicated and was ignored", weightConfig.Weight, podNamespace, svc.Name) + if weightRef == nil { + weightRef = weightConfig + weightRef.PodCount++ + hasBlueGreenDeploy = true + } else if !weightRef.Equal(weightConfig) { + glog.Warningf("deployment weight %v to service %v/%v is duplicated and was ignored", weightConfig.PodWeight, podNamespace, svc.Name) } } } else { @@ -724,12 +730,77 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) } else { glog.Warningf("could not calc weight of pod %v/%v: %v", podNamespace, podName, err) } - // weight wasn't assigned, set as zero to remove all the traffic - // without removing from the balancer - if weight < 0 { - weight = 0 + ep.WeightRef = weightRef + if weightRef != nil { + ep.Weight = weightRef.PodWeight + } else { + // weight wasn't assigned, set as zero to remove all the traffic + // without removing from the balancer + ep.Weight = 0 + } + } + if !hasBlueGreenDeploy || upstream.BlueGreen.Mode == "pod" { + // if not hasBlueGreenDeploy, nothing more to do + // if Mode == "pod", weight is already correct + continue + } + // At this moment ep.Weight refers to every single pod instead of the blue and green groups. + // Now recalc based on the number of pods on each group. + lcmPodCount := 0 + for _, weightConfig := range deployWeight { + if weightConfig.PodCount == 0 { + continue + } + podCount := weightConfig.PodCount + if lcmPodCount > 0 { + lcmPodCount = utils.LCM(lcmPodCount, podCount) + } else { + lcmPodCount = podCount + } + } + if lcmPodCount == 0 { + // all PodCount are zero, this config won't be used + continue + } + gcdGroupWeight := 0 + maxWeight := 0 + for _, weightConfig := range deployWeight { + if weightConfig.PodCount == 0 || weightConfig.PodWeight == 0 { + continue + } + groupWeight := weightConfig.PodWeight * lcmPodCount / weightConfig.PodCount + if gcdGroupWeight > 0 { + gcdGroupWeight = utils.GCD(gcdGroupWeight, groupWeight) + } else { + gcdGroupWeight = groupWeight + } + if groupWeight > maxWeight { + maxWeight = groupWeight + } + } + if gcdGroupWeight == 0 { + // all PodWeight are zero, no need to rebalance + continue + } + // HAProxy weight must be between 0..256. + // weightFactor has how many times the max weight is greater than 256. + weightFactor := float32(maxWeight) / float32(gcdGroupWeight) / float32(256) + // LCM of denominators and GCD of the results are known. Updating ep.Weight + for epID := range upstream.Endpoints { + ep := &upstream.Endpoints[epID] + if ep.WeightRef != nil { + wRef := ep.WeightRef + w := wRef.PodWeight * lcmPodCount / wRef.PodCount / gcdGroupWeight + if weightFactor > 1 { + propWeight := int(float32(w) / weightFactor) + if propWeight == 0 && wRef.PodWeight > 0 { + propWeight = 1 + } + ep.Weight = propWeight + } else { + ep.Weight = w + } } - ep.Weight = weight } } diff --git a/pkg/common/ingress/types.go b/pkg/common/ingress/types.go index 1af54dece..a76ffe97e 100644 --- a/pkg/common/ingress/types.go +++ b/pkg/common/ingress/types.go @@ -242,7 +242,8 @@ type Endpoint struct { // zero means the server will not receive any request. // Note that this is a per-endpoint configuration, so different // number of replicas need to be adjusted accordingly. - Weight int `json:"weight"` + Weight int `json:"weight"` + WeightRef *bluegreen.DeployWeight // MaxFails returns the number of unsuccessful attempts to communicate // allowed before this should be considered down. // Setting 0 indicates that the check is performed by a Kubernetes probe diff --git a/pkg/common/utils/utils.go b/pkg/common/utils/utils.go index 8d41403e2..76544b464 100644 --- a/pkg/common/utils/utils.go +++ b/pkg/common/utils/utils.go @@ -66,3 +66,17 @@ func SplitMin(str string, sub string, min int) []string { } return minSlice } + +// GCD calculates the Greatest Common Divisor between a and b +func GCD(a, b int) int { + for b != 0 { + r := a % b + a, b = b, r + } + return a +} + +// LCM calculates the Least Common Multiple between a and b +func LCM(a, b int) int { + return a * (b / GCD(a, b)) +} diff --git a/pkg/common/utils/utils_test.go b/pkg/common/utils/utils_test.go index 97ad54697..6dd02776f 100644 --- a/pkg/common/utils/utils_test.go +++ b/pkg/common/utils/utils_test.go @@ -84,3 +84,49 @@ func TestSliceMin(t *testing.T) { } } } + +func TestGCD(t *testing.T) { + testCases := []struct { + a int + b int + expected int + }{ + {10, 1, 1}, + {10, 3, 1}, + {10, 4, 2}, + {10, 5, 5}, + {10, 10, 10}, + {10, 12, 2}, + {10, 15, 5}, + {10, 20, 10}, + } + for _, test := range testCases { + res := GCD(test.a, test.b) + if res != test.expected { + t.Errorf("expected %v from %v and %v, but was %v", test.expected, test.a, test.b, res) + } + } +} + +func TestLCM(t *testing.T) { + testCases := []struct { + a int + b int + expected int + }{ + {10, 1, 10}, + {10, 3, 30}, + {10, 4, 20}, + {10, 5, 10}, + {10, 10, 10}, + {10, 12, 60}, + {10, 15, 30}, + {10, 20, 20}, + } + for _, test := range testCases { + res := LCM(test.a, test.b) + if res != test.expected { + t.Errorf("expected %v from %v and %v, but was %v", test.expected, test.a, test.b, res) + } + } +} From 2ba07b0114a3cfde158aff6099bb7b4ce4208226 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 5 Aug 2018 15:43:09 -0300 Subject: [PATCH 2/4] Doc compatibility with v0.6 --- examples/blue-green/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/blue-green/README.md b/examples/blue-green/README.md index e3897eb22..9849ffaac 100644 --- a/examples/blue-green/README.md +++ b/examples/blue-green/README.md @@ -8,7 +8,9 @@ on HAProxy Ingress controller. This document has the following prerequisite: -* A Kubernetes cluster with a running HAProxy Ingress controller. See the [five minutes deployment](/examples/setup-cluster.md#five-minutes-deployment) or the [deployment example](/examples/deployment) +* A Kubernetes cluster with a running HAProxy Ingress controller v0.6 or above. +See the [five minutes deployment](/examples/setup-cluster.md#five-minutes-deployment) +or the [deployment example](/examples/deployment) ## Deploying applications @@ -74,7 +76,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: annotations: - ingress.kubernetes.io/blue-green-balance: group=blue=1,group=green=1 + ingress.kubernetes.io/blue-green-deploy: group=blue=1,group=green=1 ingress.kubernetes.io/blue-green-mode: pod ingress.kubernetes.io/ssl-redirect: "false" name: bluegreen @@ -169,7 +171,7 @@ Changing now the balance to 1/3 blue and 2/3 green: ``` $ kubectl annotate --overwrite ingress bluegreen \ - ingress.kubernetes.io/blue-green-balance=group=blue=1,group=green=2 + ingress.kubernetes.io/blue-green-deploy=group=blue=1,group=green=2 ``` * BG Mode: deploy From d2ad4c2417dfd4c7c76db9597e7c03325caab14e Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 26 Aug 2018 18:36:37 -0300 Subject: [PATCH 3/4] Blue/green deployment tests --- pkg/common/ingress/controller/bluegreen.go | 147 ++++++++++++++++ .../ingress/controller/bluegreen_test.go | 165 ++++++++++++++++++ pkg/common/ingress/controller/controller.go | 120 +------------ 3 files changed, 313 insertions(+), 119 deletions(-) create mode 100644 pkg/common/ingress/controller/bluegreen.go create mode 100644 pkg/common/ingress/controller/bluegreen_test.go diff --git a/pkg/common/ingress/controller/bluegreen.go b/pkg/common/ingress/controller/bluegreen.go new file mode 100644 index 000000000..390b2d6c3 --- /dev/null +++ b/pkg/common/ingress/controller/bluegreen.go @@ -0,0 +1,147 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/golang/glog" + + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/store" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/utils" +) + +func weightBalance(upstreams *map[string]*ingress.Backend, podLister store.PodLister) { + // calc deployment weight based on blue/green config or draining state + for _, upstream := range *upstreams { + svc := upstream.Service + podNamespace := svc.Namespace + deployWeight := upstream.BlueGreen.DeployWeight + hasBlueGreenDeploy := false + for epID := range upstream.Endpoints { + ep := &upstream.Endpoints[epID] + if ep.Draining { + // draining state always set Weight to 0, independent of a blue/green config + ep.Weight = 0 + continue + } + if len(deployWeight) == 0 { + // no blue/green config, using default Weight config as 1 and skipping to the next + ep.Weight = 1 + continue + } + if ep.Target == nil { + glog.Warningf("ignoring blue/green config due to empty object reference on endpoint %v/%v", podNamespace, upstream.Name) + ep.Weight = 1 + continue + } + podName := ep.Target.Name + var weightRef *bluegreen.DeployWeight + if pod, err := podLister.GetPod(podNamespace, podName); err == nil { + for wID := range deployWeight { + weightConfig := &deployWeight[wID] + if label, found := pod.Labels[weightConfig.LabelName]; found { + if label == weightConfig.LabelValue { + if weightRef == nil { + weightRef = weightConfig + weightRef.PodCount++ + hasBlueGreenDeploy = true + } else if !weightRef.Equal(weightConfig) { + glog.Warningf("deployment weight %v to service %v/%v is duplicated and was ignored", weightConfig.PodWeight, podNamespace, svc.Name) + } + } + } else { + glog.Warningf("pod %v/%v does not have label %v used on blue/green deployment", podNamespace, podName, weightConfig.LabelName) + } + } + } else { + glog.Warningf("could not calc weight of pod %v/%v: %v", podNamespace, podName, err) + } + ep.WeightRef = weightRef + if weightRef != nil { + ep.Weight = weightRef.PodWeight + } else { + // weight wasn't assigned, set as zero to remove all the traffic + // without removing from the balancer + ep.Weight = 0 + } + } + if !hasBlueGreenDeploy || upstream.BlueGreen.Mode == "pod" { + // if not hasBlueGreenDeploy, nothing more to do + // if Mode == "pod", weight is already correct + continue + } + // At this moment ep.Weight refers to every single pod instead of the blue and green groups. + // Now recalc based on the number of pods on each group. + lcmPodCount := 0 + for _, weightConfig := range deployWeight { + if weightConfig.PodCount == 0 { + continue + } + podCount := weightConfig.PodCount + if lcmPodCount > 0 { + lcmPodCount = utils.LCM(lcmPodCount, podCount) + } else { + lcmPodCount = podCount + } + } + if lcmPodCount == 0 { + // all PodCount are zero, this config won't be used + continue + } + gcdGroupWeight := 0 + maxWeight := 0 + for _, weightConfig := range deployWeight { + if weightConfig.PodCount == 0 || weightConfig.PodWeight == 0 { + continue + } + groupWeight := weightConfig.PodWeight * lcmPodCount / weightConfig.PodCount + if gcdGroupWeight > 0 { + gcdGroupWeight = utils.GCD(gcdGroupWeight, groupWeight) + } else { + gcdGroupWeight = groupWeight + } + if groupWeight > maxWeight { + maxWeight = groupWeight + } + } + if gcdGroupWeight == 0 { + // all PodWeight are zero, no need to rebalance + continue + } + // HAProxy weight must be between 0..256. + // weightFactor has how many times the max weight is greater than 256. + weightFactor := float32(maxWeight) / float32(gcdGroupWeight) / float32(256) + // LCM of denominators and GCD of the results are known. Updating ep.Weight + for epID := range upstream.Endpoints { + ep := &upstream.Endpoints[epID] + if ep.WeightRef != nil { + wRef := ep.WeightRef + w := wRef.PodWeight * lcmPodCount / wRef.PodCount / gcdGroupWeight + if weightFactor > 1 { + propWeight := int(float32(w) / weightFactor) + if propWeight == 0 && wRef.PodWeight > 0 { + propWeight = 1 + } + ep.Weight = propWeight + } else { + ep.Weight = w + } + } + } + } +} diff --git a/pkg/common/ingress/controller/bluegreen_test.go b/pkg/common/ingress/controller/bluegreen_test.go new file mode 100644 index 000000000..704d56e21 --- /dev/null +++ b/pkg/common/ingress/controller/bluegreen_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strconv" + "strings" + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/store" +) + +func TestWeightBalance(t *testing.T) { + s := cache.NewStore(cache.MetaNamespaceKeyFunc) + s.Add(buildPod("pod0101-01", "app=d01,v=1")) + s.Add(buildPod("pod0101-02", "app=d01,v=1")) + s.Add(buildPod("pod0102-01", "app=d01,v=2")) + s.Add(buildPod("pod0102-02", "app=d01,v=2")) + s.Add(buildPod("pod0102-03", "app=d01,v=2")) + s.Add(buildPod("pod0102-04", "app=d01,v=2")) + s.Add(buildPod("pod0103-01", "app=d01,v=3")) + podLister := store.PodLister{ + Store: s, + } + testUpstreams := map[string]*ingress.Backend{ + "b01-01": buildBackend("v=1=50,v=2=50", "pod0101-01,pod0102-01", "deploy"), + "b01-02": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01", "deploy"), + "b01-03": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01/d", "deploy"), + "b02-01": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01,pod0102-02", "deploy"), + "b02-02": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01,pod0102-02/d", "deploy"), + "b02-03": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01,pod0102-02", "pod"), + "b02-04": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01,pod0102-02/d", "pod"), + "b02-05": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01,pod0102-02", ""), + "b02-06": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-01,pod0102-02/d", ""), + "b03-01": buildBackend("v=1=500,v=2=1", "pod0101-01,pod0102-01", "deploy"), + "b04-01": buildBackend("v=1=60,v=2=3", "pod0101-01,pod0102-01,pod0102-02,pod0102-03,pod0102-04", "deploy"), + "b04-02": buildBackend("v=1=70,v=2=3", "pod0101-01,pod0102-01,pod0102-02,pod0102-03,pod0102-04", "deploy"), + "b05-01": buildBackend("", "pod0101-01,pod0102-01", "deploy"), + "b06-01": buildBackend("v=1=50,v=2=25", ",pod0102-01", "deploy"), + "b07-01": buildBackend("v=1=50,v=non=25", "pod0101-01,pod0102-01", "deploy"), + "b07-02": buildBackend("v=1=50,v=non=25", "pod0101-01,pod0102-01", "pod"), + "b07-03": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-non", "deploy"), + "b07-04": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-non", "pod"), + "b08-01": buildBackend("v=1=50,v=2=25,v=3=25", "pod0101-01,pod0102-01,pod0102-02,pod0103-01", "deploy"), + "b08-02": buildBackend("v=1=50,v=2=0,v=3=25", "pod0101-01,pod0102-01,pod0102-02,pod0103-01", "deploy"), + "b09-01": buildBackend("v=1=50,v=2=0,v=3=25", "", "deploy"), + } + testExpectedWeight := map[string][]int{ + "b01-01": {1, 1}, + "b01-02": {2, 1}, + "b01-03": {1, 0}, + "b02-01": {4, 1, 1}, + "b02-02": {2, 1, 0}, + "b02-03": {50, 25, 25}, + "b02-04": {50, 25, 0}, + "b02-05": {4, 1, 1}, + "b02-06": {2, 1, 0}, + "b03-01": {256, 1}, + "b04-01": {80, 1, 1, 1, 1}, + "b04-02": {256, 2, 2, 2, 2}, + "b05-01": {1, 1}, + "b06-01": {1, 1}, + "b07-01": {1, 0}, + "b07-02": {50, 0}, + "b07-03": {1, 0}, + "b07-04": {50, 0}, + "b08-01": {4, 1, 1, 2}, + "b08-02": {2, 0, 0, 1}, + "b09-01": {}, + } + weightBalance(&testUpstreams, podLister) + for name, upstream := range testUpstreams { + expected := testExpectedWeight[name] + if len(upstream.Endpoints) != len(expected) { + t.Errorf("len mismatch on %v, mock: %v, expected: %v", name, len(upstream.Endpoints), len(expected)) + } + for id, ep := range upstream.Endpoints { + if ep.Weight != expected[id] { + t.Errorf("weight differs on %v[%v], real: %v, expected: %v", name, id, ep.Weight, expected[id]) + } + } + } +} + +func buildBackend(deployWeight, endpoints, bgMode string) *ingress.Backend { + w := []bluegreen.DeployWeight{} + for _, weight := range strings.Split(deployWeight, ",") { + if weight == "" { + continue + } + dwSplit := strings.Split(weight, "=") + pw, _ := strconv.ParseInt(dwSplit[2], 10, 0) + w = append(w, bluegreen.DeployWeight{ + LabelName: dwSplit[0], + LabelValue: dwSplit[1], + PodWeight: int(pw), + }) + } + ep := []ingress.Endpoint{} + if endpoints != "" { + for _, e := range strings.Split(endpoints, ",") { + epSplit := strings.Split(e, "/") + name := epSplit[0] + draining := len(epSplit) > 1 && epSplit[1] == "d" + var target *v1.ObjectReference + if name != "" { + target = &v1.ObjectReference{ + Name: name, + } + } + ep = append(ep, ingress.Endpoint{ + Draining: draining, + Target: target, + }) + } + } + return &ingress.Backend{ + Service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc01", + Namespace: "default", + }, + }, + BlueGreen: bluegreen.Config{ + DeployWeight: w, + Mode: bgMode, + }, + Endpoints: ep, + } +} + +func buildPod(name, labels string) *v1.Pod { + l := make(map[string]string) + for _, label := range strings.Split(labels, ",") { + kv := strings.Split(label, "=") + l[kv[0]] = kv[1] + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: l, + }, + } +} diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index cbb7fc56b..bf79e83ce 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -43,7 +43,6 @@ import ( "k8s.io/client-go/util/flowcontrol" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" - "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/class" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/healthcheck" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/parser" @@ -685,124 +684,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) } } - // calc deployment weight based on blue/green config or draining state - for _, upstream := range upstreams { - svc := upstream.Service - podNamespace := svc.Namespace - deployWeight := upstream.BlueGreen.DeployWeight - hasBlueGreenDeploy := false - for epID := range upstream.Endpoints { - ep := &upstream.Endpoints[epID] - if ep.Draining { - // draining state always set Weight to 0, independent of a blue/green config - ep.Weight = 0 - continue - } - if len(deployWeight) == 0 { - // no blue/green config, using default Weight config as 1 and skipping to the next - ep.Weight = 1 - continue - } - if ep.Target == nil { - glog.Warningf("ignoring blue/green config due to empty object reference on endpoint %v/%v", podNamespace, upstream.Name) - ep.Weight = 1 - continue - } - podName := ep.Target.Name - var weightRef *bluegreen.DeployWeight - if pod, err := ic.listers.Pod.GetPod(podNamespace, podName); err == nil { - for wID := range deployWeight { - weightConfig := &deployWeight[wID] - if label, found := pod.Labels[weightConfig.LabelName]; found { - if label == weightConfig.LabelValue { - if weightRef == nil { - weightRef = weightConfig - weightRef.PodCount++ - hasBlueGreenDeploy = true - } else if !weightRef.Equal(weightConfig) { - glog.Warningf("deployment weight %v to service %v/%v is duplicated and was ignored", weightConfig.PodWeight, podNamespace, svc.Name) - } - } - } else { - glog.Warningf("pod %v/%v does not have label %v used on blue/green deployment", podNamespace, podName, weightConfig.LabelName) - } - } - } else { - glog.Warningf("could not calc weight of pod %v/%v: %v", podNamespace, podName, err) - } - ep.WeightRef = weightRef - if weightRef != nil { - ep.Weight = weightRef.PodWeight - } else { - // weight wasn't assigned, set as zero to remove all the traffic - // without removing from the balancer - ep.Weight = 0 - } - } - if !hasBlueGreenDeploy || upstream.BlueGreen.Mode == "pod" { - // if not hasBlueGreenDeploy, nothing more to do - // if Mode == "pod", weight is already correct - continue - } - // At this moment ep.Weight refers to every single pod instead of the blue and green groups. - // Now recalc based on the number of pods on each group. - lcmPodCount := 0 - for _, weightConfig := range deployWeight { - if weightConfig.PodCount == 0 { - continue - } - podCount := weightConfig.PodCount - if lcmPodCount > 0 { - lcmPodCount = utils.LCM(lcmPodCount, podCount) - } else { - lcmPodCount = podCount - } - } - if lcmPodCount == 0 { - // all PodCount are zero, this config won't be used - continue - } - gcdGroupWeight := 0 - maxWeight := 0 - for _, weightConfig := range deployWeight { - if weightConfig.PodCount == 0 || weightConfig.PodWeight == 0 { - continue - } - groupWeight := weightConfig.PodWeight * lcmPodCount / weightConfig.PodCount - if gcdGroupWeight > 0 { - gcdGroupWeight = utils.GCD(gcdGroupWeight, groupWeight) - } else { - gcdGroupWeight = groupWeight - } - if groupWeight > maxWeight { - maxWeight = groupWeight - } - } - if gcdGroupWeight == 0 { - // all PodWeight are zero, no need to rebalance - continue - } - // HAProxy weight must be between 0..256. - // weightFactor has how many times the max weight is greater than 256. - weightFactor := float32(maxWeight) / float32(gcdGroupWeight) / float32(256) - // LCM of denominators and GCD of the results are known. Updating ep.Weight - for epID := range upstream.Endpoints { - ep := &upstream.Endpoints[epID] - if ep.WeightRef != nil { - wRef := ep.WeightRef - w := wRef.PodWeight * lcmPodCount / wRef.PodCount / gcdGroupWeight - if weightFactor > 1 { - propWeight := int(float32(w) / weightFactor) - if propWeight == 0 && wRef.PodWeight > 0 { - propWeight = 1 - } - ep.Weight = propWeight - } else { - ep.Weight = w - } - } - } - } + weightBalance(&upstreams, ic.listers.Pod) aUpstreams := make([]*ingress.Backend, 0, len(upstreams)) // create the list of upstreams and skip those without endpoints From c26c9cf53e78cbe33ea88745cde734d1617cbdd3 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 26 Aug 2018 19:05:19 -0300 Subject: [PATCH 4/4] Default blue/green mode as deploy; docs --- CHANGELOG.md | 8 ++++++++ README.md | 2 +- examples/blue-green/README.md | 2 +- pkg/common/ingress/annotations/bluegreen/main.go | 4 ++-- pkg/common/ingress/controller/bluegreen_test.go | 10 +++++++--- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da53777d4..e6d707a89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Current snapshot tag (v0.7) +Breaking backward compatibility from `v0.6` + +* Default blue/green deployment mode changed from `pod` to `deploy`. Use `ingress.kubernetes.io/blue-green-mode` annotation to change to the v0.6 behavior. See also the blue/green deployment [doc](/README.md#blue-green). + Fixes and improvements since `v0.6` * Add SSL config on TCP services [#192](https://github.com/jcmoraisjr/haproxy-ingress/pull/192) - [doc](/README.md#tcp-services-configmap) @@ -36,6 +40,10 @@ Fixes and improvements since `v0.6` * Configmap options: * `http-port` * `https-port` +* Add blue/green balance mode [#201](https://github.com/jcmoraisjr/haproxy-ingress/pull/201) - [doc](/README.md#blue-green) + * Annotations: + * `ingress.kubernetes.io/blue-green-balance` + * `ingress.kubernetes.io/blue-green-mode` ## v0.6-beta.2 diff --git a/README.md b/README.md index 576b65c13..7532bd797 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ selector and another that matches the blue/green selector. * `ingress.kubernetes.io/blue-green-balance`: comma separated list of labels and weights * `ingress.kubernetes.io/blue-green-deploy`: deprecated on v0.7, this is an alias to `ingress.kubernetes.io/blue-green-balance`. -* `ingress.kubernetes.io/blue-green-mode`: how to apply the weights, might be `pod` or `deploy` +* `ingress.kubernetes.io/blue-green-mode`: defaults to `deploy` on v0.7, defines how to apply the weights, might be `pod` or `deploy` The following configuration `group=blue=1,group=green=4` will redirect 20% of the load to the `group=blue` group and 80% of the load to `group=green` group. diff --git a/examples/blue-green/README.md b/examples/blue-green/README.md index 9849ffaac..0773e15a5 100644 --- a/examples/blue-green/README.md +++ b/examples/blue-green/README.md @@ -144,7 +144,7 @@ Running 100 requests... --- -Changing to deploy mode. This mode targets the balance config to the whole deployment +Changing to *deploy* mode. This mode targets the balance config to the whole deployment instead of single pods. **Note:** BG mode was added on v0.7. On v0.6, the only supported mode is `pod`. diff --git a/pkg/common/ingress/annotations/bluegreen/main.go b/pkg/common/ingress/annotations/bluegreen/main.go index 65c4ad93a..01b4a0b37 100644 --- a/pkg/common/ingress/annotations/bluegreen/main.go +++ b/pkg/common/ingress/annotations/bluegreen/main.go @@ -99,9 +99,9 @@ func (bg bgdeploy) Parse(ing *extensions.Ingress) (interface{}, error) { mode, _ := parser.GetStringAnnotation(blueGreenModeAnn, ing) if !modeAnnRegex.MatchString(mode) { if mode != "" { - glog.Warningf("unsupported blue/green mode '%v' on '%v/%v', falling back to 'pod'", mode, ing.Namespace, ing.Name) + glog.Warningf("unsupported blue/green mode '%v' on '%v/%v', falling back to 'deploy'", mode, ing.Namespace, ing.Name) } - mode = "pod" + mode = "deploy" } return &Config{ DeployWeight: dw, diff --git a/pkg/common/ingress/controller/bluegreen_test.go b/pkg/common/ingress/controller/bluegreen_test.go index 704d56e21..e11185e71 100644 --- a/pkg/common/ingress/controller/bluegreen_test.go +++ b/pkg/common/ingress/controller/bluegreen_test.go @@ -59,11 +59,13 @@ func TestWeightBalance(t *testing.T) { "b06-01": buildBackend("v=1=50,v=2=25", ",pod0102-01", "deploy"), "b07-01": buildBackend("v=1=50,v=non=25", "pod0101-01,pod0102-01", "deploy"), "b07-02": buildBackend("v=1=50,v=non=25", "pod0101-01,pod0102-01", "pod"), - "b07-03": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-non", "deploy"), - "b07-04": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-non", "pod"), + "b07-03": buildBackend("v=1=50,non=2=25", "pod0101-01,pod0102-01", "deploy"), + "b07-04": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-non", "deploy"), + "b07-05": buildBackend("v=1=50,v=2=25", "pod0101-01,pod0102-non", "pod"), "b08-01": buildBackend("v=1=50,v=2=25,v=3=25", "pod0101-01,pod0102-01,pod0102-02,pod0103-01", "deploy"), "b08-02": buildBackend("v=1=50,v=2=0,v=3=25", "pod0101-01,pod0102-01,pod0102-02,pod0103-01", "deploy"), "b09-01": buildBackend("v=1=50,v=2=0,v=3=25", "", "deploy"), + "b10-01": buildBackend("v=1=0,v=2=0", "pod0101-01,pod0102-01", "deploy"), } testExpectedWeight := map[string][]int{ "b01-01": {1, 1}, @@ -83,10 +85,12 @@ func TestWeightBalance(t *testing.T) { "b07-01": {1, 0}, "b07-02": {50, 0}, "b07-03": {1, 0}, - "b07-04": {50, 0}, + "b07-04": {1, 0}, + "b07-05": {50, 0}, "b08-01": {4, 1, 1, 2}, "b08-02": {2, 0, 0, 1}, "b09-01": {}, + "b10-01": {0, 0}, } weightBalance(&testUpstreams, podLister) for name, upstream := range testUpstreams {