Implementing a generic Ingress based on Lua And A/B Testing Release

.lift.toml

@@ -0,0 +1,5 @@
+# Ignore results from vendor directories
+ignoreFiles = """
+vendor/
+lua_configuration/
+"""

Dockerfile

@@ -20,6 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY lua_configuration /lua_configuration
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

Dockerfile_multiarch

@@ -21,6 +21,7 @@ RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GOOS=linux GOARCH=amd64
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY lua_configuration /lua_configuration
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

api/v1alpha1/rollout_types.go

@@ -20,6 +20,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
@@ -65,11 +66,6 @@ type WorkloadRef struct {
 	Name string `json:"name"`
 }
 
-/*type ControllerRevisionRef struct {
-	TargetRevisionName string `json:"targetRevisionName"`
-	SourceRevisionName string `json:"sourceRevisionName"`
-}*/
-
 // RolloutStrategy defines strategy to apply during next rollout
 type RolloutStrategy struct {
 	// Paused indicates that the Rollout is paused.
@@ -116,7 +112,19 @@ type CanaryStep struct {
 	// Pause defines a pause stage for a rollout, manual or auto
 	// +optional
 	Pause RolloutPause `json:"pause,omitempty"`
-	// MetricsAnalysis *RolloutAnalysis `json:"metricsAnalysis,omitempty"`
+	// Matches define conditions used for matching the incoming HTTP requests to canary service.
+	// Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied.
+	// If Gateway API, current only support one match.
+	// And cannot support both weight and matches, if both are configured, then matches takes precedence.
+	Matches []HttpRouteMatch `json:"matches,omitempty"`
+}
+
+type HttpRouteMatch struct {
+	// Headers specifies HTTP request header matchers. Multiple match values are
+	// ANDed together, meaning, a request must match all the specified headers
+	// to select the route.
+	// +kubebuilder:validation:MaxItems=16
+	Headers []gatewayv1alpha2.HTTPHeaderMatch `json:"headers,omitempty"`
 }
 
 // RolloutPause defines a pause stage for a rollout
@@ -141,7 +149,8 @@ type TrafficRouting struct {
 
 // IngressTrafficRouting configuration for ingress controller to control traffic routing
 type IngressTrafficRouting struct {
-	// ClassType refers to the class type of an `Ingress`, e.g. Nginx. Default is Nginx
+	// ClassType refers to the type of `Ingress`.
+	// current support nginx, aliyun-alb. default is nginx.
 	// +optional
 	ClassType string `json:"classType,omitempty"`
 	// Name refers to the name of an `Ingress` resource in the same namespace as the `Rollout`

config/crd/bases/rollouts.kruise.io_rollouts.yaml

@@ -108,6 +108,77 @@ spec:
                         items:
                           description: CanaryStep defines a step of a canary workload.
                           properties:
+                            matches:
+                              description: Matches define conditions used for matching
+                                the incoming HTTP requests to canary service. Each
+                                match is independent, i.e. this rule will be matched
+                                if **any** one of the matches is satisfied. If Gateway
+                                API, current only support one match. And cannot support
+                                both weight and matches, if both are configured, then
+                                matches takes precedence.
+                              items:
+                                properties:
+                                  headers:
+                                    description: Headers specifies HTTP request header
+                                      matchers. Multiple match values are ANDed together,
+                                      meaning, a request must match all the specified
+                                      headers to select the route.
+                                    items:
+                                      description: HTTPHeaderMatch describes how to
+                                        select a HTTP route by matching HTTP request
+                                        headers.
+                                      properties:
+                                        name:
+                                          description: "Name is the name of the HTTP
+                                            Header to be matched. Name matching MUST
+                                            be case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).
+                                            \n If multiple entries specify equivalent
+                                            header names, only the first entry with
+                                            an equivalent name MUST be considered
+                                            for a match. Subsequent entries with an
+                                            equivalent header name MUST be ignored.
+                                            Due to the case-insensitivity of header
+                                            names, \"foo\" and \"Foo\" are considered
+                                            equivalent. \n When a header is repeated
+                                            in an HTTP request, it is implementation-specific
+                                            behavior as to how this is represented.
+                                            Generally, proxies should follow the guidance
+                                            from the RFC: https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2
+                                            regarding processing a repeated header,
+                                            with special handling for \"Set-Cookie\"."
+                                          maxLength: 256
+                                          minLength: 1
+                                          pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$
+                                          type: string
+                                        type:
+                                          default: Exact
+                                          description: "Type specifies how to match
+                                            against the value of the header. \n Support:
+                                            Core (Exact) \n Support: Custom (RegularExpression)
+                                            \n Since RegularExpression HeaderMatchType
+                                            has custom conformance, implementations
+                                            can support POSIX, PCRE or any other dialects
+                                            of regular expressions. Please read the
+                                            implementation's documentation to determine
+                                            the supported dialect."
+                                          enum:
+                                          - Exact
+                                          - RegularExpression
+                                          type: string
+                                        value:
+                                          description: Value is the value of HTTP
+                                            Header to be matched.
+                                          maxLength: 4096
+                                          minLength: 1
+                                          type: string
+                                      required:
+                                      - name
+                                      - value
+                                      type: object
+                                    maxItems: 16
+                                    type: array
+                                type: object
+                              type: array
                             pause:
                               description: Pause defines a pause stage for a rollout,
                                 manual or auto
@@ -164,8 +235,9 @@ spec:
                                 to route traffic, e.g. Nginx, Alb.
                               properties:
                                 classType:
-                                  description: ClassType refers to the class type
-                                    of an `Ingress`, e.g. Nginx. Default is Nginx
+                                  description: ClassType refers to the type of `Ingress`.
+                                    current support nginx, aliyun-alb. default is
+                                    nginx.
                                   type: string
                                 name:
                                   description: Name refers to the name of an `Ingress`

config/rbac/role.yaml

@@ -144,6 +144,14 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

go.mod

@@ -4,10 +4,12 @@ go 1.16
 
 require (
 	github.com/davecgh/go-spew v1.1.1
+	github.com/evanphx/json-patch v4.11.0+incompatible
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
 	github.com/openkruise/kruise-api v1.0.0
 	github.com/spf13/pflag v1.0.5
+	github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.22.6
 	k8s.io/apiextensions-apiserver v0.22.6
@@ -16,6 +18,7 @@ require (
 	k8s.io/component-base v0.22.6
 	k8s.io/klog/v2 v2.10.0
 	k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e
+	layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf
 	sigs.k8s.io/controller-runtime v0.10.3
 	sigs.k8s.io/gateway-api v0.4.3
 	sigs.k8s.io/yaml v1.2.0

go.sum

@@ -501,6 +501,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ=
+github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@@ -675,6 +677,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1031,6 +1034,8 @@ k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e h1:ldQh+neBabomh7+89dTpiFAB8tGdfVmuIzAHbvtl+9I=
 k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:rRz0YsF7VXj9fXRF6yQgFI7DzST+hsI3TeFSGupntu0=
+layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:ivKkcY8Zxw5ba0jldhZCYYQfGdb2K6u9tbYK1AwMIBc=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

lua_configuration/trafficrouting_ingress/aliyun-alb.lua

@@ -0,0 +1,35 @@
+annotations = {}
+if ( obj.annotations )
+then
+    annotations = obj.annotations
+end
+annotations["alb.ingress.kubernetes.io/canary"] = "true"
+annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
+annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
+annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
+annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
+annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
+if ( obj.weight ~= "-1" )
+then
+    annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
+end
+if ( not obj.matches )
+then
+    return annotations
+end
+for _,match in ipairs(obj.matches) do
+    local header = match.headers[1]
+    if ( header.name == "canary-by-cookie" )
+    then
+        annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = header.value
+    else
+        annotations["alb.ingress.kubernetes.io/canary-by-header"] = header.name
+        if ( header.type == "RegularExpression" )
+        then
+            annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+        else
+            annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = header.value
+        end
+    end
+end
+return annotations

lua_configuration/trafficrouting_ingress/nginx.lua

@@ -0,0 +1,46 @@
+annotations = {}
+-- obj.annotations is ingress annotations, it is recommended not to remove the part of the lua script, it must be kept
+if ( obj.annotations )
+then
+    annotations = obj.annotations
+end
+-- indicates the ingress is nginx canary api
+annotations["nginx.ingress.kubernetes.io/canary"] = "true"
+-- First, set all canary api to nil
+annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
+-- if rollout.spec.strategy.canary.steps.weight is nil, obj.weight will be -1,
+-- then we need remove the canary-weight annotation
+if ( obj.weight ~= "-1" )
+then
+    annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
+end
+-- if don't contains headers, immediate return annotations
+if ( not obj.matches )
+then
+    return annotations
+end
+-- headers & cookie apis
+-- traverse matches
+for _,match in ipairs(obj.matches) do
+    local header = match.headers[1]
+    -- cookie
+    if ( header.name == "canary-by-cookie" )
+    then
+        annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
+    else
+        annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
+        -- if regular expression
+        if ( header.type == "RegularExpression" )
+        then
+            annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+        else
+            annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
+        end
+    end
+end
+-- must be return annotations
+return annotations

pkg/controller/rollout/rollout_controller.go

@@ -74,6 +74,7 @@ type RolloutReconciler struct {
 //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

pkg/controller/rollout/status.go

@@ -99,6 +99,8 @@ func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout
 			newStatus.Phase = rolloutv1alpha1.RolloutPhaseProgressing
 			cond := util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing")
 			util.SetRolloutCondition(&newStatus, *cond)
+			newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{}
+			newStatus.Message = "Rollout is in Progressing"
 		} else if newStatus.CanaryStatus == nil {
 			// The following logic is to make PaaS be able to judge whether the rollout is ready
 			// at the first deployment of the Rollout/Workload. For example: generally, a PaaS