Merge pull request #201 from jcmoraisjr/jm-blue-green-mode

Add blue/green balance mode
jcmoraisjr · Sep 30, 2018 · 318dfc1 · 318dfc1
2 parents f2f47fa + c26c9cf
commit 318dfc1
Show file tree

Hide file tree

Showing 10 changed files with 568 additions and 95 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## Current snapshot tag (v0.7)
 
+Breaking backward compatibility from `v0.6`
+
+* Default blue/green deployment mode changed from `pod` to `deploy`. Use `ingress.kubernetes.io/blue-green-mode` annotation to change to the v0.6 behavior. See also the blue/green deployment [doc](/README.md#blue-green).
+
 Fixes and improvements since `v0.6`
 
 * Add SSL config on TCP services [#192](https://github.com/jcmoraisjr/haproxy-ingress/pull/192) - [doc](/README.md#tcp-services-configmap)
@@ -36,6 +40,10 @@ Fixes and improvements since `v0.6`
   * Configmap options:
     * `http-port`
     * `https-port`
+* Add blue/green balance mode [#201](https://github.com/jcmoraisjr/haproxy-ingress/pull/201) - [doc](/README.md#blue-green)
+  * Annotations:
+    * `ingress.kubernetes.io/blue-green-balance`
+    * `ingress.kubernetes.io/blue-green-mode`
 
 ## v0.6-beta.3
 

diff --git a/README.md b/README.md
@@ -60,6 +60,8 @@ The following annotations are supported:
 ||[`ingress.kubernetes.io/auth-tls-secret`](#auth-tls)|namespace/secret name|[doc](/examples/auth/client-certs)|
 |`[0]`|[`ingress.kubernetes.io/balance-algorithm`](#balance-algorithm)|algorithm name|-|
 |`[0]`|[`ingress.kubernetes.io/blue-green-deploy`](#blue-green)|label=value=weight,...|[doc](/examples/blue-green)|
+|`[1]`|[`ingress.kubernetes.io/blue-green-balance`](#blue-green)|label=value=weight,...|[doc](/examples/blue-green)|
+|`[1]`|[`ingress.kubernetes.io/blue-green-mode`](#blue-green)|[pod\|deploy]|[doc](/examples/blue-green)|
 |`[0]`|[`ingress.kubernetes.io/config-backend`](#configuration-snippet)|multiline HAProxy backend config|-|
 |`[0]`|[`ingress.kubernetes.io/cors-allow-origin`](#cors)|URL|-|
 |`[0]`|[`ingress.kubernetes.io/cors-allow-methods`](#cors)|methods list|-|
@@ -132,25 +134,34 @@ See also client cert [sample](/examples/auth/client-certs).
 
 Configure weight of a blue/green deployment. The annotation accepts a comma separated list of label
 name/value pair and a numeric weight. Concatenate label name, label value and weight with an equal
-sign, without spaces. The label name/value pair will be used to match corresponding pods.
+sign, without spaces. The label name/value pair will be used to match corresponding pods or deploys.
+There is no limit to the number of label/weight balance configurations.
 
 The endpoints of a single backend are selected using service selectors, which also uses labels.
 Because of that, in order to use blue/green deployment, the deployment, daemon set or replication
 controller template should have at least two label name/value pairs - one that matches the service
 selector and another that matches the blue/green selector.
 
+* `ingress.kubernetes.io/blue-green-balance`: comma separated list of labels and weights
+* `ingress.kubernetes.io/blue-green-deploy`: deprecated on v0.7, this is an alias to `ingress.kubernetes.io/blue-green-balance`.
+* `ingress.kubernetes.io/blue-green-mode`: defaults to `deploy` on v0.7, defines how to apply the weights, might be `pod` or `deploy`
+
 The following configuration `group=blue=1,group=green=4` will redirect 20% of the load to the
-`group=blue` pods and 80% of the load to the `group=green` if they have the same number of replicas.
+`group=blue` group and 80% of the load to `group=green` group.
+
+Applying the weights depends on the blue/green mode. v0.6 has only `pod` mode which means that
+every single pod receives the same weight as configured on blue/green balance. This means that
+a balance configuration with 50% to each group will redirect twice as much requests to a backend
+that has the double of replicas. v0.7 has also `deploy` mode which rebalance the weights based
+on the number of replicas of each deployment.
 
-Note that this configuration is related to every single pod. On the configuration above, if
-`group=blue` has two replicas and `group=green` has just one, green would receive only the double
-of the number of requests dedicated to blue. This can be adjusted using higher numbers - eg `10/40`
-instead of `1/4` - and divided by the number of replicas of each deployment - eg `5/40` instead of
-`10/40`.
+In short, regarding blue/green mode: use `pod` if you want to redirect more requests to a
+deployment updating the number of replicas; use `deploy` if you want to control the load
+of each side updating the blue/green balance annotation.
 
-Value of `0` (zero) can also be used. This will let the endpoint configured in the backend accepting
-persistent connections - see [affinity](#affinity) - but will not participate in the load balancing.
-The maximum weight value is `256`.
+Value of `0` (zero) can also be used as weight. This will let the endpoint configured in the
+backend accepting persistent connections - see [affinity](#affinity) - but will not participate
+in the load balancing. The maximum weight value is `256`.
 
 See also the [example](/examples/blue-green) page.
 

diff --git a/examples/blue-green/README.md b/examples/blue-green/README.md
@@ -8,7 +8,9 @@ on HAProxy Ingress controller.
 
 This document has the following prerequisite:
 
-* A Kubernetes cluster with a running HAProxy Ingress controller. See the [five minutes deployment](/examples/setup-cluster.md#five-minutes-deployment) or the [deployment example](/examples/deployment)
+* A Kubernetes cluster with a running HAProxy Ingress controller v0.6 or above.
+See the [five minutes deployment](/examples/setup-cluster.md#five-minutes-deployment)
+or the [deployment example](/examples/deployment)
 
 ## Deploying applications
 
@@ -20,17 +22,17 @@ and `group` label as the blue/green deployment selector:
 
 ```
 $ kubectl run blue \
-  --image=gcr.io/google_containers/echoserver:1.3 \
-  --port=8080 --labels=run=bluegreen,group=blue
+  --image=jcmoraisjr/whoami \
+  --port=8000 --labels=run=bluegreen,group=blue
 deployment "blue" created
 
 $ kubectl run green \
-  --image=gcr.io/google_containers/echoserver:1.3 \
-  --port=8080 --labels=run=bluegreen,group=green
+  --image=jcmoraisjr/whoami \
+  --port=8000 --labels=run=bluegreen,group=green
 deployment "green" created
 ```
 
-Certify if the pods are running and have the correct labels. Note that both `group` and `run`
+Certify that the pods are running and have the correct labels. Note that both `group` and `run`
 labels were applied:
 
 ```
@@ -40,7 +42,7 @@ blue-79c9b67d5b-5hd2r    1/1       Running   0          35s       group=blue,pod
 green-7546d648c4-p7pmz   1/1       Running   0          28s       group=green,pod-template-hash=3102820470,run=bluegreen
 ```
 
-# Configure
+## Configure
 
 Create a service that bind both deployments together using the `run` label. The expose command need
 a deployment object, take anyone, we will override it's selector:
@@ -58,57 +60,147 @@ Check also the endpoints, it should list both blue and green pods:
 ```
 $ kubectl get ep bluegreen
 NAME         ENDPOINTS                           AGE
-bluegreen    172.17.0.11:8080,172.17.0.19:8080   2m
+bluegreen    172.17.0.11:8000,172.17.0.19:8000   2m
 
 $ kubectl get pod -lrun=bluegreen -owide
 NAME                     READY     STATUS    RESTARTS   AGE       IP            NODE
 blue-79c9b67d5b-5hd2r    1/1       Running   0          2m        172.17.0.11   192.168.100.99
 green-7546d648c4-p7pmz   1/1       Running   0          2m        172.17.0.19   192.168.100.99
 ```
 
-Configure the ingress resource:
+Configure the ingress resource. No need to change the host below, `bluegreen.example.com` is fine:
 
 ```
 $ kubectl create -f - <<EOF
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
   annotations:
+    ingress.kubernetes.io/blue-green-deploy: group=blue=1,group=green=1
+    ingress.kubernetes.io/blue-green-mode: pod
     ingress.kubernetes.io/ssl-redirect: "false"
-    ingress.kubernetes.io/blue-green-deploy: group=blue=1,group=green=4
   name: bluegreen
 spec:
   rules:
-  - host: example.com
+  - host: bluegreen.example.com
     http:
       paths:
       - backend:
           serviceName: bluegreen
-          servicePort: 8080
+          servicePort: 8000
         path: /
 EOF
 ```
 
 ```
 $ kubectl get ing
-NAME        HOSTS         ADDRESS   PORTS     AGE
-bluegreen   example.com             80        11s
+NAME        HOSTS                   ADDRESS   PORTS     AGE
+bluegreen   bluegreen.example.com             80        11s
 ```
 
-Lets test! Change the IP below to your HAProxy Ingress controller:
+# Test
+
+Lets test! The following snippets use an alias `hareq` declared below.
+Change `IP` to your HAProxy Ingress controller IP address:
+
+```
+$ IP=192.168.100.99
+$ alias hareq='echo Running 100 requests...; for i in `seq 1 100`; do
+    curl -fsS $IP -H "Host: bluegreen.example.com" | cut -d- -f1
+  done | sort | uniq -c'
+```
+
+* BG Mode: pod
+* BG Balance: blue=1, green=1
+* Replicas: blue=1, green=1
+
+```
+$ hareq
+Running 100 requests...
+  50 blue
+  50 green
+```
+
+---
+
+Now changing green replicas to 3 and wait all the replicas to be running.
+BG Mode is pod, so the number of replicas will increase the load of the green deployment.
+
+```
+$ kubectl scale deploy green --replicas=3
+$ kubectl get pod -w
+```
+
+* BG Mode: pod
+* BG Balance: blue=1, green=1
+* Replicas: blue=1, green=3
+
+```
+$ hareq
+Running 100 requests...
+  25 blue
+  75 green
+```
+
+---
+
+Changing to *deploy* mode. This mode targets the balance config to the whole deployment
+instead of single pods.
+
+**Note:** BG mode was added on v0.7. On v0.6, the only supported mode is `pod`.
 
 ```
-$ for i in `seq 1 50`; do
-    echo -n "."
-    curl -s 192.168.100.99 -H 'Host: example.com' >/dev/null
-  done
-..................................................
+$ kubectl annotate --overwrite ingress bluegreen \
+  ingress.kubernetes.io/blue-green-mode=deploy
 ```
 
-Now check `<ingress-ip>:1936` - the stats page has a `<namespace>-bluegreen-8080` card
-with backend stats. The column Session/Total should have about 20% of the deployments
-on one endpoint and 80% on another.
+* BG Mode: deploy
+* BG Balance: blue=1, green=1
+* Replicas: blue=1, green=3
 
-The blue/green configuration is related to every single pod - if the number of replicas change,
-the load will also change between blue and green groups. Have a look at the
-[doc](/README.md#blue-green) on how it works.
+```
+$ hareq
+Running 100 requests...
+  50 blue
+  50 green
+```
+
+---
+
+Changing now the balance to 1/3 blue and 2/3 green:
+
+```
+$ kubectl annotate --overwrite ingress bluegreen \
+  ingress.kubernetes.io/blue-green-deploy=group=blue=1,group=green=2
+```
+
+* BG Mode: deploy
+* BG Balance: blue=1, green=2
+* Replicas: blue=1, green=3
+
+```
+$ hareq
+Running 100 requests...
+  33 blue
+  67 green
+```
+
+---
+
+The balance will be the same despite the number of replicas:
+
+```
+$ kubectl scale deploy green --replicas=6
+$ kubectl get pod -w
+```
+
+* BG Mode: deploy
+* BG Balance: blue=1, green=2
+* Replicas: blue=1, green=6
+
+```
+$ hareq
+Running 100 requests...
+  33 blue
+  67 green
+```
diff --git a/pkg/common/ingress/annotations/bluegreen/main.go b/pkg/common/ingress/annotations/bluegreen/main.go
@@ -18,26 +18,37 @@ package bluegreen
 
 import (
 	"fmt"
+	"github.com/golang/glog"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/parser"
 	extensions "k8s.io/api/extensions/v1beta1"
+	"regexp"
 	"strconv"
 	"strings"
 )
 
 const (
-	blueGreenAnn = "ingress.kubernetes.io/blue-green-deploy"
+	blueGreenBalanceAnn = "ingress.kubernetes.io/blue-green-balance"
+	blueGreenDeployAnn  = "ingress.kubernetes.io/blue-green-deploy"
+	blueGreenModeAnn    = "ingress.kubernetes.io/blue-green-mode"
+)
+
+var (
+	modeAnnRegex = regexp.MustCompile(`^(pod|deploy)$`)
 )
 
 // DeployWeight has one label name/value pair and it's weight
 type DeployWeight struct {
-	LabelName  string
-	LabelValue string
-	Weight     int
+	LabelName   string
+	LabelValue  string
+	PodWeight   int
+	PodCount    int
+	GroupWeight int
 }
 
 // Config is the blue/green deployment configuration
 type Config struct {
 	DeployWeight []DeployWeight
+	Mode         string
 }
 
 type bgdeploy struct {
@@ -50,9 +61,12 @@ func NewParser() parser.IngressAnnotation {
 
 // Parse parses blue/green annotation and create a Config struct
 func (bg bgdeploy) Parse(ing *extensions.Ingress) (interface{}, error) {
-	s, err := parser.GetStringAnnotation(blueGreenAnn, ing)
+	s, err := parser.GetStringAnnotation(blueGreenBalanceAnn, ing)
 	if err != nil {
-		return nil, err
+		s, _ = parser.GetStringAnnotation(blueGreenDeployAnn, ing)
+		if s == "" {
+			return nil, err
+		}
 	}
 	weights := strings.Split(s, ",")
 	var dw []DeployWeight
@@ -66,22 +80,40 @@ func (bg bgdeploy) Parse(ing *extensions.Ingress) (interface{}, error) {
 			return nil, fmt.Errorf("error reading blue/green config: %v", err)
 		}
 		if w < 0 {
+			glog.Warningf("invalid weight '%v' on '%v/%v', using '0'", w, ing.Namespace, ing.Name)
 			w = 0
 		}
+		if w > 256 {
+			glog.Warningf("invalid weight '%v' on '%v/%v', using '256'", w, ing.Namespace, ing.Name)
+			w = 256
+		}
 		dwItem := DeployWeight{
-			LabelName:  dwSlice[0],
-			LabelValue: dwSlice[1],
-			Weight:     int(w),
+			LabelName:   dwSlice[0],
+			LabelValue:  dwSlice[1],
+			PodWeight:   int(w),
+			PodCount:    0, // updated in the controller
+			GroupWeight: 0, // updated in the controller
 		}
 		dw = append(dw, dwItem)
 	}
+	mode, _ := parser.GetStringAnnotation(blueGreenModeAnn, ing)
+	if !modeAnnRegex.MatchString(mode) {
+		if mode != "" {
+			glog.Warningf("unsupported blue/green mode '%v' on '%v/%v', falling back to 'deploy'", mode, ing.Namespace, ing.Name)
+		}
+		mode = "deploy"
+	}
 	return &Config{
 		DeployWeight: dw,
+		Mode:         mode,
 	}, nil
 }
 
 // Equal tests equality between two Config objects
 func (b1 *Config) Equal(b2 *Config) bool {
+	if b1.Mode != b2.Mode {
+		return false
+	}
 	if len(b1.DeployWeight) != len(b2.DeployWeight) {
 		return false
 	}
@@ -108,7 +140,7 @@ func (dw1 *DeployWeight) Equal(dw2 *DeployWeight) bool {
 	if dw1.LabelValue != dw2.LabelValue {
 		return false
 	}
-	if dw1.Weight != dw2.Weight {
+	if dw1.PodWeight != dw2.PodWeight {
 		return false
 	}
 	return true